@trillboards/edge-sdk 0.2.138 → 0.2.150

package/deploy/detector/trillboards_detector/server.py

@@ -33,7 +33,7 @@ import threading
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 from ._version import VERSION
-from .errors import DetectorNotReady, UnknownClasses
+from .errors import DetectorNotReady, ModelVerificationError, UnknownClasses
 
 #: Loopback only. litert-lm owns 9379; this is a SECOND service, not a route on
 #: that one — see `LocalDetectorClient.ts` for why (its own uv tool, its own
@@ -126,6 +126,19 @@ def parse_detect_request(raw: bytes) -> tuple[bytes, float, "list[str] | None"]:
     return jpeg, min_confidence, raw_classes
 
 
+def parse_expected_model(raw: bytes):
+    payload = json.loads(raw.decode("utf-8"))
+    if "expected_model" not in payload:
+        return None
+    value = payload["expected_model"]
+    if (not isinstance(value, dict) or set(value) != {"modelId", "sha256"}
+        or not isinstance(value["modelId"], str) or not value["modelId"]
+        or not isinstance(value["sha256"], str) or len(value["sha256"]) != 64
+        or any(c not in "0123456789abcdef" for c in value["sha256"])):
+        raise WireError(400, "expected_model requires one modelId and a lowercase SHA-256")
+    return value
+
+
 def _assert_wire_safe(detections: list) -> None:
     """The client's fifteen refusals, checked on THIS side before answering 200.
 
@@ -204,6 +217,12 @@ def build_detect_response(batch) -> dict:
     if not isinstance(batch.inference_ms, (int, float)) or not math.isfinite(batch.inference_ms):
         raise ValueError(f"inference_ms is not a finite number: {batch.inference_ms!r}")
     _assert_wire_safe(body["detections"])
+    hashes = getattr(batch, "model_hashes", {})
+    if hashes:
+        if (not isinstance(hashes, dict) or set(hashes) != set(models)
+            or any(not isinstance(v, str) or len(v) != 64 or any(c not in "0123456789abcdef" for c in v) for v in hashes.values())):
+            raise ValueError("loaded model hash provenance is incomplete")
+        body["model_hashes"] = hashes
     return body
 
 
@@ -283,6 +302,7 @@ class DetectorHandler(BaseHTTPRequestHandler):
         try:
             raw = self._read_body()
             jpeg, min_confidence, requested = parse_detect_request(raw)
+            expected_model = parse_expected_model(raw)
         except WireError as err:
             self._json(err.status, {"error": err.message})
             return
@@ -307,7 +327,11 @@ class DetectorHandler(BaseHTTPRequestHandler):
                 return
 
         try:
-            batch = loader.detect(jpeg, min_confidence, wanted)
+            batch = (loader.detect(jpeg, min_confidence, wanted) if expected_model is None
+                     else loader.detect(jpeg, min_confidence, wanted, expected_model=expected_model))
+        except ModelVerificationError as err:
+            self._json(409, {"error": str(err)})
+            return
         except DetectorNotReady as err:
             # 503, never a 200 with an empty array. The client raises
             # DetectorUnavailableError on this and falls through to the VLM.
@@ -320,6 +344,9 @@ class DetectorHandler(BaseHTTPRequestHandler):
 
         try:
             body = build_detect_response(batch)
+            if expected_model is not None and (body.get("model_hashes") != {expected_model["modelId"]: expected_model["sha256"]}
+                                              or body["model"] != expected_model["modelId"]):
+                raise ValueError("selected detector did not return its loaded artifact proof")
         except ValueError as err:
             self._log(f"[detector] REFUSING to emit a malformed payload: {err}")
             self._json(500, {"error": f"refused to emit a malformed payload: {err}"})

package/deploy/detector/trillboards_detector/session.py

@@ -29,6 +29,7 @@ and fails on any disagreement.
 from __future__ import annotations
 
 import io
+import hashlib
 import math
 import threading
 import time
@@ -49,6 +50,7 @@ from .decode import (
 )
 from .detections import Detection, DetectionBatch
 from .errors import ModelContractError
+from .execution import LocalOrtSession
 from .labels import ModelVocabulary
 
 #: `rfdetr/detr.py` — ImageNet statistics, applied AFTER scaling to [0,1].
@@ -349,6 +351,7 @@ class OnnxDetector:
         model_path: Path,
         vocabulary: ModelVocabulary,
         intra_op_threads: "int | None" = None,
+        accelerator: str = "auto",
     ):
         self.model_path = model_path
         self.vocabulary = vocabulary
@@ -364,9 +367,12 @@ class OnnxDetector:
         # multiply an already non-monotonic pool by the number of models.
         opts.log_severity_level = 3
 
-        self._session = ort.InferenceSession(
-            str(model_path), sess_options=opts, providers=["CPUExecutionProvider"]
-        )
+        # Loading and hashing the same immutable bytes avoids attributing an
+        # old resident session to an OTA file which replaced its original path.
+        model_bytes = Path(model_path).read_bytes()
+        self.model_sha256 = hashlib.sha256(model_bytes).hexdigest()
+        self._session = LocalOrtSession(ort, model_bytes, opts, accelerator)
+        del model_bytes
         self.spec = _read_graph_spec(self._session, vocabulary)
 
     def detect(
@@ -391,6 +397,7 @@ class OnnxDetector:
         with self._lock:
             infer_started = time.perf_counter()
             outputs = self._session.run(list(self.spec.output_names), {self.spec.input_name: blob})
+            self.spec.providers = list(self._session.get_providers())
             infer_ms = (time.perf_counter() - infer_started) * 1000.0
 
         if layout == RFDETR_DETS_LABELS:
@@ -421,8 +428,13 @@ class OnnxDetector:
 
         batch.inference_ms = infer_ms
         batch.total_ms = (time.perf_counter() - started) * 1000.0
+        batch.model_hashes = {self.model_name: self.model_sha256}
         return batch
 
+    @property
+    def acceleration(self):
+        return self._session.acceleration
+
 
 def _read_graph_spec(
     session: "ort.InferenceSession",

package/deploy/litertlm/test_trillboards_litertlm_shim.py

@@ -6,11 +6,14 @@ the recorded kwargs prove WHICH configs are dropped and which survive.
 """
 
 import importlib
+import hashlib
 import os
 import sys
 import tempfile
 import textwrap
 import unittest
+from pathlib import Path
+from types import SimpleNamespace
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
@@ -127,6 +130,84 @@ class ShimTest(unittest.TestCase):
         sys.meta_path.insert(0, object())
         self.assertIn(shim.install(), (True, False))
 
+    def test_loaded_snapshot_survives_replacement_of_the_original_model(self):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        source = Path(self.tmp.name) / "model.litertlm"
+        source.write_bytes(b"evaluated local model")
+        snapshot, sha = shim._verified_snapshot(source)
+        source.write_bytes(b"later OTA model")
+        self.assertEqual(Path(snapshot).read_bytes(), b"evaluated local model")
+        self.assertEqual(sha, hashlib.sha256(b"evaluated local model").hexdigest())
+
+    def test_modified_snapshot_is_refused(self):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        source = Path(self.tmp.name) / "model.litertlm"
+        source.write_bytes(b"evaluated local model")
+        snapshot, _ = shim._verified_snapshot(source)
+        Path(snapshot).write_bytes(b"tampered")
+        with self.assertRaisesRegex(ValueError, "snapshot was modified"):
+            shim._verified_snapshot(source)
+
+    def _runtime_handler(self, expected, artifact=None, actual_backend="gpu"):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        engine = SimpleNamespace(backend=SimpleNamespace(get_name=lambda: actual_backend))
+        if artifact is not None:
+            engine._trillboards_artifact_sha256 = artifact
+        captured = {}
+
+        class Handler:
+            headers = {"X-Trillboards-Model-Sha256": expected} if expected is not None else {}
+
+            def send_error(self, status, message):
+                captured["error"] = (status, message)
+
+            def _get_engine(self, model_id, *, backend):
+                return engine
+
+            def do_POST(self):
+                selected = self._get_engine("gemma", backend="gpu")
+                if selected is not None:
+                    module._dump_json({"object": "chat.completion", "choices": [{"text": "unchanged"}]})
+
+        def dump(data, **kwargs):
+            captured["response"] = data
+
+        module = SimpleNamespace(OpenAIHandler=Handler, _dump_json=dump)
+        shim.apply_runtime_handler(module)
+        module.OpenAIHandler().do_POST()
+        return captured
+
+    def test_http_proof_is_from_the_loaded_engine_not_the_requested_name(self):
+        sha = "a" * 64
+        captured = self._runtime_handler(sha, sha)
+        self.assertEqual(captured["response"]["runtimeModel"], {
+            "version": 1, "sha256": sha, "format": "litert-lm", "runtimeModelId": "gemma,gpu",
+        })
+        self.assertEqual(captured["response"]["choices"], [{"text": "unchanged"}])
+
+    def test_missing_artifact_proof_is_not_fabricated(self):
+        captured = self._runtime_handler("a" * 64)
+        self.assertEqual(captured["error"][0], 409)
+        self.assertNotIn("response", captured)
+
+    def test_wrong_artifact_is_refused_before_answering(self):
+        captured = self._runtime_handler("a" * 64, "b" * 64)
+        self.assertEqual(captured["error"][0], 409)
+        self.assertNotIn("response", captured)
+
+    def test_wrong_backend_is_refused_before_answering(self):
+        captured = self._runtime_handler("a" * 64, "a" * 64, actual_backend="cpu")
+        self.assertEqual(captured["error"][0], 409)
+
+    def test_malformed_requested_hash_is_refused(self):
+        captured = self._runtime_handler("not-a-hash", "a" * 64)
+        self.assertEqual(captured["error"][0], 400)
+
+    def test_a_request_without_pinning_still_works_without_fabricated_proof(self):
+        captured = self._runtime_handler(None)
+        self.assertEqual(captured["response"]["choices"], [{"text": "unchanged"}])
+        self.assertNotIn("runtimeModel", captured["response"])
+
 
 if __name__ == "__main__":
     unittest.main()

package/deploy/litertlm/test_verify_model_proof.py

@@ -0,0 +1,30 @@
+import unittest
+
+from verify_model_proof import verify_completion
+
+
+class ModelProofTest(unittest.TestCase):
+    def proof(self, model_id="gemma,gpu", sha="a" * 64):
+        return {"runtimeModel": {"version": 1, "format": "litert-lm",
+                                 "runtimeModelId": model_id, "sha256": sha}}
+
+    def test_exact_gpu(self):
+        self.assertTrue(verify_completion(self.proof(), "gemma,gpu", "a" * 64))
+
+    def test_cpu_normalization(self):
+        self.assertTrue(verify_completion(self.proof("gemma,cpu"), "gemma", "a" * 64))
+
+    def test_gpu_cannot_demote_to_cpu(self):
+        self.assertFalse(verify_completion(self.proof("gemma,cpu"), "gemma,gpu", "a" * 64))
+
+    def test_wrong_file_backend_and_alias(self):
+        for payload in [self.proof(sha="b" * 64), self.proof("gemma,cpu"), self.proof("other,gpu")]:
+            self.assertFalse(verify_completion(payload, "gemma,gpu", "a" * 64))
+
+    def test_no_proof_cannot_be_replaced_by_model_text(self):
+        for payload in [None, {}, {"choices": [{"message": {"content": "correct model"}}]}]:
+            self.assertFalse(verify_completion(payload, "gemma,gpu", "a" * 64))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/deploy/litertlm/trillboards_litertlm_shim.py

@@ -67,7 +67,9 @@ imports nothing, so it piggybacks on the import the CLI was going to do anyway
 rather than adding one. `litert-lm list` and `litert-lm --version` — which the
 provisioners run as health checks — keep exactly the startup cost they had.
 
-It also fails OPEN. Every failure path leaves stock behaviour intact and prints
+The conversation-speed optimization fails OPEN. A pinned identity request does
+not: it fails CLOSED when the loaded artifact or requested backend is unproven.
+An optimization failure leaves stock behaviour intact and prints
 one line to stderr, which the serve's own log captures: a serve that is merely
 slow keeps sensing, and a serve that refuses to start senses nothing. The line
 is the only way to tell the two apart from the outside, so it is not optional —
@@ -127,6 +129,8 @@ def apply_to(litert_lm_module: object) -> bool:
     Returns True when the wrap is in place (including when it already was).
     """
     engine_cls = getattr(litert_lm_module, "Engine", None)
+    if engine_cls is not None:
+        _apply_model_snapshot(engine_cls)
     original = getattr(engine_cls, "create_conversation", None)
     if original is None:
         _say("NOT ACTIVE: litert_lm.Engine.create_conversation is absent — "
@@ -170,6 +174,8 @@ class _LitertLmImportHook:
             sys.meta_path.remove(self)
         except ValueError:
             return None
+        if not _enabled():
+            return None
         try:
             import importlib.util
 
@@ -198,6 +204,7 @@ def install() -> bool:
         if not _enabled():
             _say("disabled by TRILLBOARDS_LITERTLM_SHIM=0")
             return False
+        _install_runtime_handler_hook()
         already = sys.modules.get(TARGET_MODULE)
         if already is not None:
             _install_now(already)
@@ -211,4 +218,146 @@ def install() -> bool:
         return False
 
 
+def _verified_snapshot(model_path):
+    """Copy/hash once at engine load. Inference reads these immutable bytes.
+
+    Stored by content hash beside the model; OTA replaces the original path,
+    never this snapshot. No per-frame hashing or extra model inference.
+    """
+    import hashlib
+    import os
+    import tempfile
+    from pathlib import Path
+    source = Path(model_path)
+    directory = source.parent / 'verified-runtime'
+    directory.mkdir(exist_ok=True)
+    digest = hashlib.sha256()
+    fd, temporary = tempfile.mkstemp(prefix='.loading-', dir=directory)
+    try:
+        with os.fdopen(fd, 'wb') as target, source.open('rb') as original:
+            for chunk in iter(lambda: original.read(1024 * 1024), b''):
+                digest.update(chunk)
+                target.write(chunk)
+        sha = digest.hexdigest()
+        snapshot = directory / (sha + source.suffix)
+        if snapshot.exists():
+            check = hashlib.sha256()
+            with snapshot.open('rb') as existing:
+                for chunk in iter(lambda: existing.read(1024 * 1024), b''):
+                    check.update(chunk)
+            if check.hexdigest() != sha:
+                raise ValueError('verified model snapshot was modified')
+        else:
+            os.replace(temporary, snapshot)
+        return str(snapshot), sha
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def _apply_model_snapshot(engine_cls):
+    original = engine_cls.__init__
+    if getattr(original, '_trillboards_model_snapshot', False):
+        return
+
+    def initialize(self, *args, **kwargs):
+        path = kwargs.get('model_path', args[0] if args else None)
+        if path is None:  # lightweight contract-test engines have no artifact
+            return original(self, *args, **kwargs)
+        snapshot, sha = _verified_snapshot(path)
+        if 'model_path' in kwargs:
+            kwargs['model_path'] = snapshot
+        else:
+            args = (snapshot, *args[1:])
+        original(self, *args, **kwargs)
+        self._trillboards_artifact_sha256 = sha
+
+    initialize._trillboards_model_snapshot = True
+    engine_cls.__init__ = initialize
+
+
+def apply_runtime_handler(module):
+    """Bind proof to the engine used by this HTTP request, outside model text."""
+    import contextvars
+    import re
+    handler = getattr(module, 'OpenAIHandler', None)
+    if handler is None or getattr(handler, '_trillboards_runtime_identity', False):
+        return
+    active = contextvars.ContextVar('trillboards_runtime_identity', default=None)
+    original_post, original_engine, original_dump = handler.do_POST, handler._get_engine, module._dump_json
+
+    def do_post(self):
+        expected = self.headers.get('X-Trillboards-Model-Sha256')
+        if expected is not None and not re.fullmatch(r'[a-f0-9]{64}', expected):
+            self.send_error(400, 'invalid selected artifact hash')
+            return
+        token = active.set({'expected': expected, 'proof': None})
+        try:
+            return original_post(self)
+        finally:
+            active.reset(token)
+
+    def get_engine(self, *args, **kwargs):
+        engine = original_engine(self, *args, **kwargs)
+        state = active.get()
+        if engine is None or state is None:
+            return engine
+        sha = getattr(engine, '_trillboards_artifact_sha256', None)
+        if state['expected'] is not None and sha != state['expected']:
+            self.send_error(409, 'selected model artifact does not match loaded engine')
+            return None
+        # The CLI separates "model,gpu" into model_id and backend before this
+        # call. Reconstruct identity from the loaded engine, not just the name.
+        backend = kwargs.get('backend', args[3] if len(args) > 3 else None)
+        get_name = getattr(getattr(engine, 'backend', None), 'get_name', None)
+        actual_backend = get_name() if callable(get_name) else None
+        if state['expected'] is not None and backend is not None and actual_backend != backend:
+            self.send_error(409, 'selected backend does not match loaded engine')
+            return None
+        runtime_id = kwargs.get('model_id', args[0] if args else None)
+        if backend is not None:
+            runtime_id = f'{runtime_id},{actual_backend}'
+        if sha is not None:
+            state['proof'] = {'version': 1, 'sha256': sha, 'format': 'litert-lm',
+                'runtimeModelId': runtime_id}
+        return engine
+
+    def dump_json(data, **kwargs):
+        state = active.get()
+        if isinstance(data, dict) and data.get('object') == 'chat.completion' and state and state['proof']:
+            data = {**data, 'runtimeModel': dict(state['proof'])}
+        return original_dump(data, **kwargs)
+
+    handler.do_POST, handler._get_engine, module._dump_json = do_post, get_engine, dump_json
+    handler._trillboards_runtime_identity = True
+
+
+class _RuntimeHandlerImportHook:
+    def find_spec(self, fullname, path=None, target=None):
+        if fullname != 'litert_lm_cli.commands.openai_handler':
+            return None
+        sys.meta_path.remove(self)
+        if not _enabled():
+            return None
+        import importlib.util
+        spec = importlib.util.find_spec(fullname)
+        if spec is None or spec.loader is None:
+            return None
+        original = spec.loader.exec_module
+
+        def execute(module):
+            original(module)
+            apply_runtime_handler(module)
+        spec.loader.exec_module = execute
+        return spec
+
+
+def _install_runtime_handler_hook():
+    module = sys.modules.get('litert_lm_cli.commands.openai_handler')
+    if module is not None:
+        apply_runtime_handler(module)
+    elif not any(isinstance(hook, _RuntimeHandlerImportHook) for hook in sys.meta_path):
+        sys.meta_path.insert(0, _RuntimeHandlerImportHook())
+
+
 install()

package/deploy/litertlm/verify_model_proof.py

@@ -0,0 +1,30 @@
+"""Validate the server's engine identity, independently of model-generated text."""
+
+import json
+import sys
+
+
+def verify_completion(payload, model_id, artifact_sha256):
+    proof = payload.get("runtimeModel") if isinstance(payload, dict) else None
+    if not isinstance(proof, dict):
+        return False
+    backend = "gpu" if model_id.endswith(",gpu") else "cpu"
+    names = {model_id}
+    if backend == "cpu" and not model_id.endswith(",cpu"):
+        names.add(model_id + ",cpu")
+    return (
+        proof.get("version") == 1
+        and set(proof) == {"version", "format", "sha256", "runtimeModelId"}
+        and proof.get("format") == "litert-lm"
+        and proof.get("sha256") == artifact_sha256
+        and proof.get("runtimeModelId") in names
+    )
+
+
+if __name__ == "__main__":
+    try:
+        with open(sys.argv[1], encoding="utf-8") as response:
+            valid = verify_completion(json.load(response), sys.argv[2], sys.argv[3])
+    except (OSError, ValueError, IndexError):
+        valid = False
+    sys.exit(0 if valid else 1)