@trillboards/edge-sdk 0.2.196 → 0.2.197

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,57 @@
1
+ """Native conversation rendering and token accounting, without model inference."""
2
+ import sys
3
+ import unittest
4
+ from pathlib import Path
5
+ from types import SimpleNamespace
6
+ sys.path.insert(0, str(Path(__file__).parent))
7
+ import trillboards_litertlm_shim as shim
8
+
9
+ class TokenBudgetTest(unittest.TestCase):
10
+ def conversation(self, n=20, capacity=100, previous=3):
11
+ calls = []
12
+ def tokenize(text):
13
+ calls.append(text)
14
+ return list(range(n))
15
+ c = SimpleNamespace(_engine=SimpleNamespace(tokenize=tokenize, max_num_tokens=capacity, _trillboards_initial_prefix_tokens=1),
16
+ token_count=previous, render_message_to_string=lambda message: 'native chat template')
17
+ return c, calls
18
+
19
+ def test_uses_the_native_render_and_loaded_engine_tokenizer(self):
20
+ self.assertTrue(callable(getattr(shim, '_native_text_budget', None)))
21
+ c, calls = self.conversation()
22
+ result = shim._native_text_budget(c, {'role':'user','content':[{'type':'text','text':'x'*5000}]}, 10)
23
+ self.assertEqual(calls, ['native chat template'])
24
+ self.assertEqual(result, {'version':1,'contextTokens':100,'inputTokens':23,'reservedOutputTokens':10})
25
+
26
+ def test_output_capacity_is_reserved_before_generation(self):
27
+ self.assertTrue(callable(getattr(shim, '_native_text_budget', None)))
28
+ c, _ = self.conversation(n=90)
29
+ with self.assertRaisesRegex(ValueError, 'context|budget'):
30
+ shim._native_text_budget(c, {'role':'user','content':[]}, 10)
31
+
32
+ def test_missing_capacity_and_image_token_accounting_are_refused(self):
33
+ self.assertTrue(callable(getattr(shim, '_native_text_budget', None)))
34
+ c, _ = self.conversation(capacity=None)
35
+ with self.assertRaises(ValueError):
36
+ shim._native_text_budget(c, {'role':'user','content':[]}, 10)
37
+ c, _ = self.conversation()
38
+ with self.assertRaises(ValueError):
39
+ shim._native_text_budget(c, {'role':'user','content':[{'type':'image','blob':'x'}]}, 10)
40
+
41
+ def test_native_initial_prefix_is_counted_only_on_an_empty_conversation(self):
42
+ c, _ = self.conversation(previous=0)
43
+ self.assertEqual(shim._native_text_budget(c, {'role':'user','content':[]}, 10)['inputTokens'],21)
44
+ c.token_count = 8
45
+ self.assertEqual(shim._native_text_budget(c, {'role':'user','content':[]}, 10)['inputTokens'],28)
46
+
47
+ def test_unmeasured_prefix_cannot_pass_a_budget(self):
48
+ c, _ = self.conversation(previous=0)
49
+ del c._engine._trillboards_initial_prefix_tokens
50
+ with self.assertRaisesRegex(ValueError, 'prefix'):
51
+ shim._native_text_budget(c, {'role':'user','content':[]}, 10)
52
+
53
+ def test_prefix_measurement_is_a_native_engine_operation(self):
54
+ self.assertTrue(callable(getattr(shim, '_measure_native_token_prefix', None)))
55
+
56
+ if __name__ == '__main__':
57
+ unittest.main()
@@ -317,6 +317,63 @@ def _apply_native_images(module):
317
317
  conversation.send_message_async = send
318
318
 
319
319
 
320
+ class _ContextBudgetExceeded(ValueError):
321
+ def __init__(self, budget):
322
+ super().__init__('local context budget exceeded before generation')
323
+ self.budget = budget
324
+
325
+
326
+ def _measure_native_token_prefix(engine):
327
+ """Measure the loaded template's initial prefix once, before allocating the request conversation.
328
+
329
+ Native rendering excludes the runtime's initial special tokens. Measure that
330
+ difference using two distinct native prefills, then verify each request's
331
+ observed usage against its budget. No model-family guesses or byte ratios.
332
+ """
333
+ if hasattr(engine, '_trillboards_initial_prefix_tokens'):
334
+ return
335
+ offsets = []
336
+ for text in ('x', 'Native token accounting: café 123.'):
337
+ message = {'role': 'user', 'content': [{'type': 'text', 'text': text}]}
338
+ with engine.create_conversation(messages=[]) as probe:
339
+ rendered_tokens = len(engine.tokenize(probe.render_message_to_string(message)))
340
+ probe.send_message(message, max_output_tokens=1)
341
+ observed = probe.get_benchmark_info().last_prefill_token_count
342
+ if isinstance(observed, bool) or not isinstance(observed, int):
343
+ raise ValueError('native prefix prefill count required')
344
+ offsets.append(observed - rendered_tokens)
345
+ if offsets[0] != offsets[1] or not 0 <= offsets[0] <= 64:
346
+ raise ValueError('native initial prefix is not stable for this loaded template')
347
+ engine._trillboards_initial_prefix_tokens = offsets[0]
348
+
349
+
350
+ def _native_text_budget(conversation, message, reserve):
351
+ """Count the exact native chat rendering on its already loaded tokenizer."""
352
+ engine = getattr(conversation, '_engine', None)
353
+ capacity = getattr(engine, 'max_num_tokens', None)
354
+ if (isinstance(capacity, bool) or not isinstance(capacity, int) or capacity < 1
355
+ or isinstance(reserve, bool) or not isinstance(reserve, int) or reserve < 1):
356
+ raise ValueError('native context capacity and output budget required')
357
+ parts = message.get('content', []) if isinstance(message, dict) else []
358
+ if any(not isinstance(p, dict) or p.get('type') != 'text' for p in parts):
359
+ raise ValueError('native text token budget cannot account for image tokens')
360
+ rendered = conversation.render_message_to_string(message)
361
+ if not isinstance(rendered, str) or not rendered:
362
+ raise ValueError('native chat rendering required for token budget')
363
+ previous = conversation.token_count
364
+ if isinstance(previous, bool) or not isinstance(previous, int) or previous < 0:
365
+ raise ValueError('native conversation token count required')
366
+ prefix = getattr(engine, '_trillboards_initial_prefix_tokens', None)
367
+ if isinstance(prefix, bool) or not isinstance(prefix, int) or prefix < 0:
368
+ raise ValueError('measured native initial prefix required')
369
+ budget = {'version': 1, 'contextTokens': capacity,
370
+ 'inputTokens': previous + len(engine.tokenize(rendered)) + (prefix if previous == 0 else 0),
371
+ 'reservedOutputTokens': reserve}
372
+ if budget['inputTokens'] + reserve > capacity:
373
+ raise _ContextBudgetExceeded(budget)
374
+ return budget
375
+
376
+
320
377
  def apply_runtime_handler(module):
321
378
  """Bind proof to the engine used by this HTTP request, outside model text."""
322
379
  import contextvars
@@ -326,12 +383,17 @@ def apply_runtime_handler(module):
326
383
  return
327
384
  active = contextvars.ContextVar('trillboards_runtime_identity', default=None)
328
385
  original_post, original_engine, original_dump = handler.do_POST, handler._get_engine, module._dump_json
386
+ original_chat = getattr(handler, '_handle_chat_completions', None)
329
387
 
330
388
  def do_post(self):
331
389
  expected = self.headers.get('X-Trillboards-Model-Sha256')
332
390
  if expected is not None and not re.fullmatch(r'[a-f0-9]{64}', expected):
333
391
  self.send_error(400, 'invalid selected artifact hash')
334
392
  return
393
+ budget_header = self.headers.get('X-Trillboards-Token-Budget')
394
+ if budget_header not in (None, 'required') or budget_header is not None and expected is None:
395
+ self.send_error(400, 'pinned native token budget request required')
396
+ return
335
397
  raw_images = self.headers.get('X-Trillboards-Image-Sha256')
336
398
  images = raw_images.split(',') if raw_images is not None else None
337
399
  if images is not None and (expected is None or not 1 <= len(images) <= 5
@@ -339,7 +401,7 @@ def apply_runtime_handler(module):
339
401
  self.send_error(400, 'invalid pinned native image request')
340
402
  return
341
403
  image_token = _image_request.set({'expected': images, 'consumed': None} if images is not None else None)
342
- token = active.set({'expected': expected, 'proof': None})
404
+ token = active.set({'expected': expected, 'proof': None, 'budgetRequired': budget_header == 'required', 'budget': None})
343
405
  try:
344
406
  return original_post(self)
345
407
  finally:
@@ -376,15 +438,46 @@ def apply_runtime_handler(module):
376
438
  runtime_id = kwargs.get('model_id', args[0] if args else None)
377
439
  if backend is not None:
378
440
  runtime_id = f'{runtime_id},{actual_backend}'
441
+ if state.get('budgetRequired'):
442
+ try:
443
+ _measure_native_token_prefix(engine)
444
+ except Exception as error:
445
+ self.send_error(422, f'native token prefix measurement failed: {type(error).__name__}')
446
+ return None
379
447
  if sha is not None:
380
448
  state['proof'] = {'version': 1, 'sha256': sha, 'format': 'litert-lm',
381
449
  'runtimeModelId': runtime_id}
382
450
  return engine
383
451
 
452
+ def handle_chat(self, conv, prompt, *args, **kwargs):
453
+ state = active.get()
454
+ if state and state.get('budgetRequired'):
455
+ try:
456
+ state['budget'] = _native_text_budget(conv, prompt, kwargs.get('max_completion_tokens'))
457
+ except _ContextBudgetExceeded as error:
458
+ self._headers_sent = True
459
+ self.send_response(413)
460
+ self.send_header('Content-Type', 'application/json')
461
+ self.end_headers()
462
+ self.wfile.write(original_dump({'error': 'local_context_budget_exceeded',
463
+ 'tokenBudget': error.budget, 'runtimeModel': state['proof']}).encode('utf-8'))
464
+ return
465
+ except ValueError as error:
466
+ self.send_error(422, str(error))
467
+ return
468
+ return original_chat(self, conv, prompt, *args, **kwargs)
469
+
384
470
  def dump_json(data, **kwargs):
385
471
  state = active.get()
386
472
  if isinstance(data, dict) and data.get('object') == 'chat.completion' and state and state['proof']:
387
473
  data = {**data, 'runtimeModel': dict(state['proof'])}
474
+ budget = state.get('budget')
475
+ if budget is not None:
476
+ data = {**data, 'tokenBudget': budget}
477
+ usage = data.get('usage') or {}
478
+ if (usage.get('completion_tokens', 0) >= budget['reservedOutputTokens']
479
+ or usage.get('total_tokens', 0) >= budget['contextTokens']):
480
+ data = {**data, 'choices': [{**c, 'finish_reason': 'length'} for c in data.get('choices', [])]}
388
481
  images = _image_request.get()
389
482
  if isinstance(data, dict) and data.get('object') == 'chat.completion' and images is not None:
390
483
  if images['consumed'] != images['expected']:
@@ -393,6 +486,8 @@ def apply_runtime_handler(module):
393
486
  return original_dump(data, **kwargs)
394
487
 
395
488
  handler.do_POST, handler._get_engine, module._dump_json = do_post, get_engine, dump_json
489
+ if original_chat is not None:
490
+ handler._handle_chat_completions = handle_chat
396
491
  handler._trillboards_runtime_identity = True
397
492
 
398
493
 
package/dist/EdgeAgent.js CHANGED
@@ -44,7 +44,7 @@ const edge_ads_1 = require("@trillboards/edge-ads");
44
44
  const CommandRouter_1 = require("./CommandRouter");
45
45
  const config_1 = require("./config");
46
46
  const dataDir_1 = require("./dataDir");
47
- const SDK_VERSION = '0.2.196';
47
+ const SDK_VERSION = '0.2.197';
48
48
  class EdgeAgent extends events_1.EventEmitter {
49
49
  constructor(config, deps) {
50
50
  super();
@@ -31,7 +31,7 @@ export interface DiagnosticReport {
31
31
  skipped: number;
32
32
  };
33
33
  }
34
- declare const SDK_VERSION = "0.2.196";
34
+ declare const SDK_VERSION = "0.2.197";
35
35
  declare function checkNodeVersion(): CheckResult;
36
36
  declare function checkPlatform(): CheckResult;
37
37
  declare function checkMemory(): CheckResult;
package/dist/diagnose.js CHANGED
@@ -63,7 +63,7 @@ const localNetworkAccess_1 = require("./localNetworkAccess");
63
63
  // ---------------------------------------------------------------------------
64
64
  // SDK version — mirrors package.json; kept in sync by CI
65
65
  // ---------------------------------------------------------------------------
66
- const SDK_VERSION = '0.2.196';
66
+ const SDK_VERSION = '0.2.197';
67
67
  exports._SDK_VERSION = SDK_VERSION;
68
68
  // ---------------------------------------------------------------------------
69
69
  // ANSI helpers (no external deps)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@trillboards/edge-sdk",
3
- "version": "0.2.196",
3
+ "version": "0.2.197",
4
4
  "description": "Composable cross-platform Edge AI SDK for audience sensing on DOOH/CTV devices",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",