@trillboards/edge-sdk 0.2.196 → 0.2.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Native conversation rendering and token accounting, without model inference."""
|
|
2
|
+
import sys
|
|
3
|
+
import unittest
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from types import SimpleNamespace
|
|
6
|
+
sys.path.insert(0, str(Path(__file__).parent))
|
|
7
|
+
import trillboards_litertlm_shim as shim
|
|
8
|
+
|
|
9
|
+
class TokenBudgetTest(unittest.TestCase):
|
|
10
|
+
def conversation(self, n=20, capacity=100, previous=3):
|
|
11
|
+
calls = []
|
|
12
|
+
def tokenize(text):
|
|
13
|
+
calls.append(text)
|
|
14
|
+
return list(range(n))
|
|
15
|
+
c = SimpleNamespace(_engine=SimpleNamespace(tokenize=tokenize, max_num_tokens=capacity, _trillboards_initial_prefix_tokens=1),
|
|
16
|
+
token_count=previous, render_message_to_string=lambda message: 'native chat template')
|
|
17
|
+
return c, calls
|
|
18
|
+
|
|
19
|
+
def test_uses_the_native_render_and_loaded_engine_tokenizer(self):
|
|
20
|
+
self.assertTrue(callable(getattr(shim, '_native_text_budget', None)))
|
|
21
|
+
c, calls = self.conversation()
|
|
22
|
+
result = shim._native_text_budget(c, {'role':'user','content':[{'type':'text','text':'x'*5000}]}, 10)
|
|
23
|
+
self.assertEqual(calls, ['native chat template'])
|
|
24
|
+
self.assertEqual(result, {'version':1,'contextTokens':100,'inputTokens':23,'reservedOutputTokens':10})
|
|
25
|
+
|
|
26
|
+
def test_output_capacity_is_reserved_before_generation(self):
|
|
27
|
+
self.assertTrue(callable(getattr(shim, '_native_text_budget', None)))
|
|
28
|
+
c, _ = self.conversation(n=90)
|
|
29
|
+
with self.assertRaisesRegex(ValueError, 'context|budget'):
|
|
30
|
+
shim._native_text_budget(c, {'role':'user','content':[]}, 10)
|
|
31
|
+
|
|
32
|
+
def test_missing_capacity_and_image_token_accounting_are_refused(self):
|
|
33
|
+
self.assertTrue(callable(getattr(shim, '_native_text_budget', None)))
|
|
34
|
+
c, _ = self.conversation(capacity=None)
|
|
35
|
+
with self.assertRaises(ValueError):
|
|
36
|
+
shim._native_text_budget(c, {'role':'user','content':[]}, 10)
|
|
37
|
+
c, _ = self.conversation()
|
|
38
|
+
with self.assertRaises(ValueError):
|
|
39
|
+
shim._native_text_budget(c, {'role':'user','content':[{'type':'image','blob':'x'}]}, 10)
|
|
40
|
+
|
|
41
|
+
def test_native_initial_prefix_is_counted_only_on_an_empty_conversation(self):
|
|
42
|
+
c, _ = self.conversation(previous=0)
|
|
43
|
+
self.assertEqual(shim._native_text_budget(c, {'role':'user','content':[]}, 10)['inputTokens'],21)
|
|
44
|
+
c.token_count = 8
|
|
45
|
+
self.assertEqual(shim._native_text_budget(c, {'role':'user','content':[]}, 10)['inputTokens'],28)
|
|
46
|
+
|
|
47
|
+
def test_unmeasured_prefix_cannot_pass_a_budget(self):
|
|
48
|
+
c, _ = self.conversation(previous=0)
|
|
49
|
+
del c._engine._trillboards_initial_prefix_tokens
|
|
50
|
+
with self.assertRaisesRegex(ValueError, 'prefix'):
|
|
51
|
+
shim._native_text_budget(c, {'role':'user','content':[]}, 10)
|
|
52
|
+
|
|
53
|
+
def test_prefix_measurement_is_a_native_engine_operation(self):
|
|
54
|
+
self.assertTrue(callable(getattr(shim, '_measure_native_token_prefix', None)))
|
|
55
|
+
|
|
56
|
+
if __name__ == '__main__':
|
|
57
|
+
unittest.main()
|
|
@@ -317,6 +317,63 @@ def _apply_native_images(module):
|
|
|
317
317
|
conversation.send_message_async = send
|
|
318
318
|
|
|
319
319
|
|
|
320
|
+
class _ContextBudgetExceeded(ValueError):
|
|
321
|
+
def __init__(self, budget):
|
|
322
|
+
super().__init__('local context budget exceeded before generation')
|
|
323
|
+
self.budget = budget
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _measure_native_token_prefix(engine):
|
|
327
|
+
"""Measure the loaded template's initial prefix once, before allocating the request conversation.
|
|
328
|
+
|
|
329
|
+
Native rendering excludes the runtime's initial special tokens. Measure that
|
|
330
|
+
difference using two distinct native prefills, then verify each request's
|
|
331
|
+
observed usage against its budget. No model-family guesses or byte ratios.
|
|
332
|
+
"""
|
|
333
|
+
if hasattr(engine, '_trillboards_initial_prefix_tokens'):
|
|
334
|
+
return
|
|
335
|
+
offsets = []
|
|
336
|
+
for text in ('x', 'Native token accounting: café 123.'):
|
|
337
|
+
message = {'role': 'user', 'content': [{'type': 'text', 'text': text}]}
|
|
338
|
+
with engine.create_conversation(messages=[]) as probe:
|
|
339
|
+
rendered_tokens = len(engine.tokenize(probe.render_message_to_string(message)))
|
|
340
|
+
probe.send_message(message, max_output_tokens=1)
|
|
341
|
+
observed = probe.get_benchmark_info().last_prefill_token_count
|
|
342
|
+
if isinstance(observed, bool) or not isinstance(observed, int):
|
|
343
|
+
raise ValueError('native prefix prefill count required')
|
|
344
|
+
offsets.append(observed - rendered_tokens)
|
|
345
|
+
if offsets[0] != offsets[1] or not 0 <= offsets[0] <= 64:
|
|
346
|
+
raise ValueError('native initial prefix is not stable for this loaded template')
|
|
347
|
+
engine._trillboards_initial_prefix_tokens = offsets[0]
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _native_text_budget(conversation, message, reserve):
|
|
351
|
+
"""Count the exact native chat rendering on its already loaded tokenizer."""
|
|
352
|
+
engine = getattr(conversation, '_engine', None)
|
|
353
|
+
capacity = getattr(engine, 'max_num_tokens', None)
|
|
354
|
+
if (isinstance(capacity, bool) or not isinstance(capacity, int) or capacity < 1
|
|
355
|
+
or isinstance(reserve, bool) or not isinstance(reserve, int) or reserve < 1):
|
|
356
|
+
raise ValueError('native context capacity and output budget required')
|
|
357
|
+
parts = message.get('content', []) if isinstance(message, dict) else []
|
|
358
|
+
if any(not isinstance(p, dict) or p.get('type') != 'text' for p in parts):
|
|
359
|
+
raise ValueError('native text token budget cannot account for image tokens')
|
|
360
|
+
rendered = conversation.render_message_to_string(message)
|
|
361
|
+
if not isinstance(rendered, str) or not rendered:
|
|
362
|
+
raise ValueError('native chat rendering required for token budget')
|
|
363
|
+
previous = conversation.token_count
|
|
364
|
+
if isinstance(previous, bool) or not isinstance(previous, int) or previous < 0:
|
|
365
|
+
raise ValueError('native conversation token count required')
|
|
366
|
+
prefix = getattr(engine, '_trillboards_initial_prefix_tokens', None)
|
|
367
|
+
if isinstance(prefix, bool) or not isinstance(prefix, int) or prefix < 0:
|
|
368
|
+
raise ValueError('measured native initial prefix required')
|
|
369
|
+
budget = {'version': 1, 'contextTokens': capacity,
|
|
370
|
+
'inputTokens': previous + len(engine.tokenize(rendered)) + (prefix if previous == 0 else 0),
|
|
371
|
+
'reservedOutputTokens': reserve}
|
|
372
|
+
if budget['inputTokens'] + reserve > capacity:
|
|
373
|
+
raise _ContextBudgetExceeded(budget)
|
|
374
|
+
return budget
|
|
375
|
+
|
|
376
|
+
|
|
320
377
|
def apply_runtime_handler(module):
|
|
321
378
|
"""Bind proof to the engine used by this HTTP request, outside model text."""
|
|
322
379
|
import contextvars
|
|
@@ -326,12 +383,17 @@ def apply_runtime_handler(module):
|
|
|
326
383
|
return
|
|
327
384
|
active = contextvars.ContextVar('trillboards_runtime_identity', default=None)
|
|
328
385
|
original_post, original_engine, original_dump = handler.do_POST, handler._get_engine, module._dump_json
|
|
386
|
+
original_chat = getattr(handler, '_handle_chat_completions', None)
|
|
329
387
|
|
|
330
388
|
def do_post(self):
|
|
331
389
|
expected = self.headers.get('X-Trillboards-Model-Sha256')
|
|
332
390
|
if expected is not None and not re.fullmatch(r'[a-f0-9]{64}', expected):
|
|
333
391
|
self.send_error(400, 'invalid selected artifact hash')
|
|
334
392
|
return
|
|
393
|
+
budget_header = self.headers.get('X-Trillboards-Token-Budget')
|
|
394
|
+
if budget_header not in (None, 'required') or budget_header is not None and expected is None:
|
|
395
|
+
self.send_error(400, 'pinned native token budget request required')
|
|
396
|
+
return
|
|
335
397
|
raw_images = self.headers.get('X-Trillboards-Image-Sha256')
|
|
336
398
|
images = raw_images.split(',') if raw_images is not None else None
|
|
337
399
|
if images is not None and (expected is None or not 1 <= len(images) <= 5
|
|
@@ -339,7 +401,7 @@ def apply_runtime_handler(module):
|
|
|
339
401
|
self.send_error(400, 'invalid pinned native image request')
|
|
340
402
|
return
|
|
341
403
|
image_token = _image_request.set({'expected': images, 'consumed': None} if images is not None else None)
|
|
342
|
-
token = active.set({'expected': expected, 'proof': None})
|
|
404
|
+
token = active.set({'expected': expected, 'proof': None, 'budgetRequired': budget_header == 'required', 'budget': None})
|
|
343
405
|
try:
|
|
344
406
|
return original_post(self)
|
|
345
407
|
finally:
|
|
@@ -376,15 +438,46 @@ def apply_runtime_handler(module):
|
|
|
376
438
|
runtime_id = kwargs.get('model_id', args[0] if args else None)
|
|
377
439
|
if backend is not None:
|
|
378
440
|
runtime_id = f'{runtime_id},{actual_backend}'
|
|
441
|
+
if state.get('budgetRequired'):
|
|
442
|
+
try:
|
|
443
|
+
_measure_native_token_prefix(engine)
|
|
444
|
+
except Exception as error:
|
|
445
|
+
self.send_error(422, f'native token prefix measurement failed: {type(error).__name__}')
|
|
446
|
+
return None
|
|
379
447
|
if sha is not None:
|
|
380
448
|
state['proof'] = {'version': 1, 'sha256': sha, 'format': 'litert-lm',
|
|
381
449
|
'runtimeModelId': runtime_id}
|
|
382
450
|
return engine
|
|
383
451
|
|
|
452
|
+
def handle_chat(self, conv, prompt, *args, **kwargs):
|
|
453
|
+
state = active.get()
|
|
454
|
+
if state and state.get('budgetRequired'):
|
|
455
|
+
try:
|
|
456
|
+
state['budget'] = _native_text_budget(conv, prompt, kwargs.get('max_completion_tokens'))
|
|
457
|
+
except _ContextBudgetExceeded as error:
|
|
458
|
+
self._headers_sent = True
|
|
459
|
+
self.send_response(413)
|
|
460
|
+
self.send_header('Content-Type', 'application/json')
|
|
461
|
+
self.end_headers()
|
|
462
|
+
self.wfile.write(original_dump({'error': 'local_context_budget_exceeded',
|
|
463
|
+
'tokenBudget': error.budget, 'runtimeModel': state['proof']}).encode('utf-8'))
|
|
464
|
+
return
|
|
465
|
+
except ValueError as error:
|
|
466
|
+
self.send_error(422, str(error))
|
|
467
|
+
return
|
|
468
|
+
return original_chat(self, conv, prompt, *args, **kwargs)
|
|
469
|
+
|
|
384
470
|
def dump_json(data, **kwargs):
|
|
385
471
|
state = active.get()
|
|
386
472
|
if isinstance(data, dict) and data.get('object') == 'chat.completion' and state and state['proof']:
|
|
387
473
|
data = {**data, 'runtimeModel': dict(state['proof'])}
|
|
474
|
+
budget = state.get('budget')
|
|
475
|
+
if budget is not None:
|
|
476
|
+
data = {**data, 'tokenBudget': budget}
|
|
477
|
+
usage = data.get('usage') or {}
|
|
478
|
+
if (usage.get('completion_tokens', 0) >= budget['reservedOutputTokens']
|
|
479
|
+
or usage.get('total_tokens', 0) >= budget['contextTokens']):
|
|
480
|
+
data = {**data, 'choices': [{**c, 'finish_reason': 'length'} for c in data.get('choices', [])]}
|
|
388
481
|
images = _image_request.get()
|
|
389
482
|
if isinstance(data, dict) and data.get('object') == 'chat.completion' and images is not None:
|
|
390
483
|
if images['consumed'] != images['expected']:
|
|
@@ -393,6 +486,8 @@ def apply_runtime_handler(module):
|
|
|
393
486
|
return original_dump(data, **kwargs)
|
|
394
487
|
|
|
395
488
|
handler.do_POST, handler._get_engine, module._dump_json = do_post, get_engine, dump_json
|
|
489
|
+
if original_chat is not None:
|
|
490
|
+
handler._handle_chat_completions = handle_chat
|
|
396
491
|
handler._trillboards_runtime_identity = True
|
|
397
492
|
|
|
398
493
|
|
package/dist/EdgeAgent.js
CHANGED
|
@@ -44,7 +44,7 @@ const edge_ads_1 = require("@trillboards/edge-ads");
|
|
|
44
44
|
const CommandRouter_1 = require("./CommandRouter");
|
|
45
45
|
const config_1 = require("./config");
|
|
46
46
|
const dataDir_1 = require("./dataDir");
|
|
47
|
-
const SDK_VERSION = '0.2.
|
|
47
|
+
const SDK_VERSION = '0.2.197';
|
|
48
48
|
class EdgeAgent extends events_1.EventEmitter {
|
|
49
49
|
constructor(config, deps) {
|
|
50
50
|
super();
|
package/dist/diagnose.d.ts
CHANGED
|
@@ -31,7 +31,7 @@ export interface DiagnosticReport {
|
|
|
31
31
|
skipped: number;
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
|
-
declare const SDK_VERSION = "0.2.
|
|
34
|
+
declare const SDK_VERSION = "0.2.197";
|
|
35
35
|
declare function checkNodeVersion(): CheckResult;
|
|
36
36
|
declare function checkPlatform(): CheckResult;
|
|
37
37
|
declare function checkMemory(): CheckResult;
|
package/dist/diagnose.js
CHANGED
|
@@ -63,7 +63,7 @@ const localNetworkAccess_1 = require("./localNetworkAccess");
|
|
|
63
63
|
// ---------------------------------------------------------------------------
|
|
64
64
|
// SDK version — mirrors package.json; kept in sync by CI
|
|
65
65
|
// ---------------------------------------------------------------------------
|
|
66
|
-
const SDK_VERSION = '0.2.
|
|
66
|
+
const SDK_VERSION = '0.2.197';
|
|
67
67
|
exports._SDK_VERSION = SDK_VERSION;
|
|
68
68
|
// ---------------------------------------------------------------------------
|
|
69
69
|
// ANSI helpers (no external deps)
|
package/package.json
CHANGED