@trillboards/edge-sdk 0.2.61 → 0.2.63

package/deploy/litertlm/test_trillboards_litertlm_shim.py

@@ -0,0 +1,132 @@
+"""Fixture tests for the litert-lm constrained-decoding shim.
+
+Runs with no litert-lm installed: a stand-in `litert_lm` module is planted on
+sys.path so the import hook is exercised exactly as it is in production, and
+the recorded kwargs prove WHICH configs are dropped and which survive.
+"""
+
+import importlib
+import os
+import sys
+import tempfile
+import textwrap
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+
+FAKE_LITERT_LM = textwrap.dedent(
+    '''
+    class ConstrainedDecodingConfig:
+        def __init__(self, enable=False, provider=None):
+            self.enable = enable
+            self.provider = provider
+
+    class Engine:
+        last_kwargs = None
+
+        def create_conversation(self, **kwargs):
+            Engine.last_kwargs = dict(kwargs)
+            return "conversation"
+    '''
+)
+
+
+class ShimTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        with open(os.path.join(self.tmp.name, "litert_lm.py"), "w") as fh:
+            fh.write(FAKE_LITERT_LM)
+        sys.path.insert(0, self.tmp.name)
+        self.addCleanup(lambda: sys.path.remove(self.tmp.name))
+        for name in ("litert_lm", "trillboards_litertlm_shim"):
+            sys.modules.pop(name, None)
+        self.addCleanup(lambda: [sys.modules.pop(n, None) for n in ("litert_lm", "trillboards_litertlm_shim")])
+        self._meta = list(sys.meta_path)
+        self.addCleanup(lambda: sys.meta_path.__setitem__(slice(None), self._meta))
+
+    def _load(self, **env):
+        old = {k: os.environ.get(k) for k in env}
+        os.environ.update({k: v for k, v in env.items() if v is not None})
+        for k, v in env.items():
+            if v is None:
+                os.environ.pop(k, None)
+
+        def restore():
+            for k, v in old.items():
+                if v is None:
+                    os.environ.pop(k, None)
+                else:
+                    os.environ[k] = v
+
+        self.addCleanup(restore)
+        return importlib.import_module("trillboards_litertlm_shim")
+
+    # ── the rule itself ────────────────────────────────────────────────────
+    def test_drops_only_the_no_response_format_shape(self):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        import litert_lm
+
+        Cfg = litert_lm.ConstrainedDecodingConfig
+        self.assertTrue(shim.should_drop(Cfg(enable=True, provider=None)))
+        # a caller who asked for a response_format keeps their grammar
+        self.assertFalse(shim.should_drop(Cfg(enable=True, provider="LL_GUIDANCE")))
+        self.assertFalse(shim.should_drop(Cfg(enable=False, provider=None)))
+        self.assertFalse(shim.should_drop(None))
+        # truthy-but-not-True must NOT be treated as the upstream shape
+        self.assertFalse(shim.should_drop(Cfg(enable=1, provider=None)))
+
+    # ── the wrap, through a real import ────────────────────────────────────
+    def test_import_hook_wraps_and_drops(self):
+        self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        import litert_lm
+
+        cfg = litert_lm.ConstrainedDecodingConfig(enable=True, provider=None)
+        litert_lm.Engine().create_conversation(messages=[], constrained_decoding_config=cfg)
+        self.assertIsNone(litert_lm.Engine.last_kwargs["constrained_decoding_config"])
+        # every other kwarg is passed through untouched
+        self.assertEqual(litert_lm.Engine.last_kwargs["messages"], [])
+
+    def test_import_hook_preserves_a_real_response_format_request(self):
+        self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        import litert_lm
+
+        cfg = litert_lm.ConstrainedDecodingConfig(enable=True, provider="LL_GUIDANCE")
+        litert_lm.Engine().create_conversation(constrained_decoding_config=cfg)
+        self.assertIs(litert_lm.Engine.last_kwargs["constrained_decoding_config"], cfg)
+
+    def test_kill_switch_leaves_stock_behaviour(self):
+        self._load(TRILLBOARDS_LITERTLM_SHIM="0")
+        import litert_lm
+
+        cfg = litert_lm.ConstrainedDecodingConfig(enable=True, provider=None)
+        litert_lm.Engine().create_conversation(constrained_decoding_config=cfg)
+        self.assertIs(litert_lm.Engine.last_kwargs["constrained_decoding_config"], cfg)
+
+    def test_apply_is_idempotent(self):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        import litert_lm
+
+        first = litert_lm.Engine.create_conversation
+        self.assertTrue(shim.apply_to(litert_lm))
+        self.assertIs(litert_lm.Engine.create_conversation, first)
+
+    # ── fails OPEN, never closed ───────────────────────────────────────────
+    def test_missing_engine_attribute_does_not_raise(self):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+
+        class NoEngine:
+            pass
+
+        self.assertFalse(shim.apply_to(NoEngine()))
+
+    def test_install_never_raises(self):
+        shim = self._load(TRILLBOARDS_LITERTLM_SHIM="1")
+        # a hostile meta_path must not take the serve down
+        sys.meta_path.insert(0, object())
+        self.assertIn(shim.install(), (True, False))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/deploy/litertlm/trillboards-litertlm-shim.pth

@@ -0,0 +1 @@
+import trillboards_litertlm_shim

package/deploy/litertlm/trillboards_litertlm_shim.py

@@ -0,0 +1,214 @@
+"""Drop the constrained-decoding setup `litert-lm serve` pays on EVERY request.
+
+WHY THIS FILE EXISTS
+--------------------
+`litert-lm serve`'s OpenAI shim builds, once per HTTP request, at both
+`openai_handler.py` construction sites (`/v1/chat/completions` and
+`/v1/responses`, 0.15.0 / 0.16.0 / 0.16.1 alike)::
+
+    provider = LL_GUIDANCE if response_format is not None else None
+    constrained_decoding_config = ConstrainedDecodingConfig(enable=True, provider=provider)
+    with engine.create_conversation(..., constrained_decoding_config=cfg) as conv:
+
+`enable` is **unconditionally True** — including when the caller sent no
+`response_format` at all and the generation that follows is therefore
+unconstrained. Enabling it makes the native `litert_lm_conversation_create`
+build the constraint machinery for a grammar nothing will use, and it is
+rebuilt for every conversation because the serve creates one per request.
+
+MEASURED (Apple M4 Max 16c/48G, litert-lm 0.16.0, Gemma 4 E2B, GPU/Metal
+delegate, vision_backend=gpu + speculative_decoding=true from config.json, the
+two committed CCTV fixtures, operator/SOP field set, one engine, arms alternated
+sample-by-sample with the leading arm rotated each round, n=10 pairs, load
+average 22):
+
+    arm                      create_conversation   send_message   total
+    stock (enable=True)            0.717 s            0.560 s     1.275 s
+    this shim (dropped)            0.001 s            0.603 s     0.604 s
+
+    paired total ratio 2.194x (min 2.040, max 2.284)
+    generations BYTE-IDENTICAL 10/10; prefill 493 and decode 58 tokens on both
+
+`send_message` is untouched — this removes setup, not sampling — and the
+answers are the same bytes, which is the property that makes it safe.
+
+THE RATIO IS ITSELF LOAD-DEPENDENT, and that is worth knowing before quoting
+one: the same paired comparison through the shipped HTTP path gives 3.734x on a
+box at 0.0% CPU idle and 2.42x at load 25, because the work this removes is
+CPU-bound while the work it leaves is GPU-bound. The near-idle figure is the
+honest one; a number taken under contention flatters this change.
+
+The connector never sends `response_format`: LiteRtVlmProvider gets its
+structured output by asking for a bare JSON object and validating client-side,
+and it re-measured `response_format` at +35.2% p50 and DECLINED it. So on the
+shipped path this setup is pure waste, every frame.
+
+WHAT IT DOES NOT TOUCH
+----------------------
+The rule fires on EXACTLY `enable is True and provider is None` — the shape the
+shim builds when, and only when, the caller asked for no `response_format`. A
+request that DOES carry one arrives as `provider=LL_GUIDANCE` and is passed
+through unchanged, so constrained decoding still works for anyone who asks for
+it. `ConstrainedDecodingConfig` is constructed nowhere else in the litert-lm
+CLI (`run.py` and `gemini_handler.py` create conversations without one), so no
+other command can reach this wrap.
+
+HOW IT IS LOADED
+----------------
+The three provisioners copy this file plus a one-line `.pth` into the litert-lm
+tool venv's site-packages. That leaves every upstream file byte-identical on
+disk, needs no change to how the serve is launched on any platform (the macOS
+LaunchAgent, the systemd unit and the Windows Scheduled Task all still exec
+`litert-lm serve`), and covers a serve started by hand as well as one started
+by a service manager.
+
+It is LAZY on purpose: `install()` only registers a `sys.meta_path` hook and
+imports nothing, so it piggybacks on the import the CLI was going to do anyway
+rather than adding one. `litert-lm list` and `litert-lm --version` — which the
+provisioners run as health checks — keep exactly the startup cost they had.
+
+It also fails OPEN. Every failure path leaves stock behaviour intact and prints
+one line to stderr, which the serve's own log captures: a serve that is merely
+slow keeps sensing, and a serve that refuses to start senses nothing. The line
+is the only way to tell the two apart from the outside, so it is not optional —
+grep the serve log for `[trillboards-litertlm-shim]`.
+
+Set `TRILLBOARDS_LITERTLM_SHIM=0` to disable it without uninstalling anything.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+__all__ = ["install", "apply_to", "should_drop", "TAG"]
+
+TAG = "[trillboards-litertlm-shim]"
+
+#: The module whose import triggers the wrap. `Engine.create_conversation` is
+#: the single funnel every serve request goes through.
+TARGET_MODULE = "litert_lm"
+
+#: Marker set on the wrapper so a second install() is a no-op rather than a
+#: second layer of indirection.
+_MARKER = "_trillboards_litertlm_shim"
+
+
+def _say(message: str) -> None:
+    """One line to stderr. Never raises — a logging failure must not be fatal."""
+    try:
+        sys.stderr.write(f"{TAG} {message}\n")
+        sys.stderr.flush()
+    except Exception:  # pragma: no cover - stderr is gone; nothing to do
+        pass
+
+
+def _enabled() -> bool:
+    """False only when explicitly switched off, so a missing var means ON."""
+    return os.environ.get("TRILLBOARDS_LITERTLM_SHIM", "1").strip() not in ("0", "false", "no")
+
+
+def should_drop(config: object) -> bool:
+    """Is this the no-response_format shape litert-lm builds on every request?
+
+    Deliberately exact. `enable is True` and not merely truthy, `provider is
+    None` and not merely falsy: anything else is a caller who asked for
+    constrained decoding, and dropping theirs would change an answer rather
+    than a setup cost.
+    """
+    if config is None:
+        return False
+    return getattr(config, "enable", None) is True and getattr(config, "provider", "?") is None
+
+
+def apply_to(litert_lm_module: object) -> bool:
+    """Wrap `Engine.create_conversation` on an imported litert_lm. Idempotent.
+
+    Returns True when the wrap is in place (including when it already was).
+    """
+    engine_cls = getattr(litert_lm_module, "Engine", None)
+    original = getattr(engine_cls, "create_conversation", None)
+    if original is None:
+        _say("NOT ACTIVE: litert_lm.Engine.create_conversation is absent — "
+             "upstream shape changed; serving with stock (slower) behaviour")
+        return False
+    if getattr(original, _MARKER, False):
+        return True
+
+    def create_conversation(self, *args, **kwargs):
+        if should_drop(kwargs.get("constrained_decoding_config")):
+            kwargs["constrained_decoding_config"] = None
+        return original(self, *args, **kwargs)
+
+    # Not functools.wraps: the marker must survive, and copying __wrapped__
+    # would let a second install() unwrap back to the stock method.
+    create_conversation.__name__ = getattr(original, "__name__", "create_conversation")
+    create_conversation.__doc__ = getattr(original, "__doc__", None)
+    setattr(create_conversation, _MARKER, True)
+    setattr(engine_cls, "create_conversation", create_conversation)
+    return True
+
+
+def _install_now(module: object) -> None:
+    if apply_to(module):
+        _say("ACTIVE: unconstrained conversations skip constrained-decoding setup")
+
+
+class _LitertLmImportHook:
+    """Applies the wrap immediately after `litert_lm` finishes importing.
+
+    A `find_spec` that answers only for the target module, hands the real
+    machinery back the job of finding it, and decorates the resulting loader's
+    `exec_module` so the wrap lands after the module body has run. It removes
+    itself first, so it can never recurse and never slows a later import.
+    """
+
+    def find_spec(self, fullname, path=None, target=None):  # noqa: D401
+        if fullname != TARGET_MODULE:
+            return None
+        try:
+            sys.meta_path.remove(self)
+        except ValueError:
+            return None
+        try:
+            import importlib.util
+
+            spec = importlib.util.find_spec(fullname)
+            if spec is None or spec.loader is None or not hasattr(spec.loader, "exec_module"):
+                return None
+            real_exec_module = spec.loader.exec_module
+
+            def exec_module(module):
+                real_exec_module(module)
+                try:
+                    _install_now(module)
+                except Exception as exc:  # noqa: BLE001
+                    _say(f"NOT ACTIVE: {exc!r} — serving with stock (slower) behaviour")
+
+            spec.loader.exec_module = exec_module
+            return spec
+        except Exception as exc:  # noqa: BLE001
+            _say(f"NOT ACTIVE: {exc!r} — serving with stock (slower) behaviour")
+            return None
+
+
+def install() -> bool:
+    """Register the import hook. Safe to call more than once. Never raises."""
+    try:
+        if not _enabled():
+            _say("disabled by TRILLBOARDS_LITERTLM_SHIM=0")
+            return False
+        already = sys.modules.get(TARGET_MODULE)
+        if already is not None:
+            _install_now(already)
+            return True
+        if any(isinstance(f, _LitertLmImportHook) for f in sys.meta_path):
+            return True
+        sys.meta_path.insert(0, _LitertLmImportHook())
+        return True
+    except Exception as exc:  # noqa: BLE001
+        _say(f"NOT ACTIVE: {exc!r} — serving with stock (slower) behaviour")
+        return False
+
+
+install()

package/deploy/macos/bench-local-vlm.sh

@@ -6,10 +6,14 @@
 #
 # WHY IT EXISTS. Local-only inference makes the box spec the product spec, and
 # every latency number in the capacity plan came off one Apple M4 Max
-# workstation — including the 7.30 s/frame in connectorCadence.ts that sizes the
-# whole 493-analyses/hour budget. A 3-core / 7 GB Apple silicon runner is the
-# closest CI has to the Mac mini we would actually recommend, and it is the only
-# cheap way to test the "16 GB+ RAM" minimum the activation runbook publishes.
+# workstation — including the 7.30 s/frame in connectorCadence.ts, which is now
+# the BOOTSTRAP a connector uses until it has measured itself (see
+# analyzeCostObserver.ts) rather than a fleet-wide fact. A 3-core / 7 GB Apple
+# silicon runner is the closest CI has to the Mac mini we would actually
+# recommend, and its 7 GB is why it is also the cheapest evidence about memory:
+# the "16 GB+ RAM" minimum this used to point at was RETIRED on 2026-08-18 —
+# cctv-rushi-activation.md now publishes 8 GB, against a measured working set of
+# 2.7 GB stock and 3.31 GB on the shipped GPU-vision path.
 #
 # It does NOT use setup-litertlm.sh's LaunchAgent path: that bootstraps a
 # per-login agent, and a CI runner's session is not the session a customer logs

package/deploy/macos/build-macos-pkg.sh

@@ -398,6 +398,10 @@ cp "$PKG_ROOT/deploy/macos/com.trillboards.edge.plist" "$PAYLOAD/share/com.trill
 # verifies the private service before it opens camera pairing.
 cp "$PKG_ROOT/deploy/macos/com.trillboards.litertlm.plist" "$PAYLOAD/share/com.trillboards.litertlm.plist"
 cp "$PKG_ROOT/deploy/macos/scripts/setup-litertlm.sh" "$PAYLOAD/share/setup-litertlm.sh"
+# Staged NEXT TO setup-litertlm.sh because that is where install_litertlm_shim
+# looks first. It is copied into the litert-lm tool venv's site-packages at
+# provisioning time; nothing else reads it, and no upstream file is modified.
+cp "$PKG_ROOT/deploy/litertlm/trillboards_litertlm_shim.py" "$PAYLOAD/share/trillboards_litertlm_shim.py"
 cp "$PKG_ROOT/deploy/macos/scripts/ensure-litertlm.sh" "$PAYLOAD/share/ensure-litertlm.sh"
 cp "$PKG_ROOT/deploy/macos/apply-update.sh" "$PAYLOAD/share/apply-update.sh"
 chmod +x "$PAYLOAD/share/setup-litertlm.sh" "$PAYLOAD/share/ensure-litertlm.sh" \

package/deploy/macos/scripts/setup-litertlm.sh

@@ -66,8 +66,23 @@ fail() { log "ERROR: $*"; exit 1; }
 # `litert-lm serve` takes NO flags for any of this: `serve` accepts only
 # --host/--port/--api/--cors-origin/--verbose. Every engine knob it honours comes
 # from `$LITERT_LM_DIR/config.json`, which `litert_lm_cli.config.get_model_config`
-# merges per model id. So this file IS the tuning surface, and writing it is the
-# whole change -- the serve stays stock.
+# merges per model id. So this file IS the tuning surface for everything it can
+# express, and the serve binary stays stock: we pass it no flags and we modify
+# none of its files.
+#
+# ONE knob is NOT expressible there, and it is the largest one left. Every
+# request the serve handles gets `ConstrainedDecodingConfig(enable=True, ...)`
+# built for it unconditionally -- `enable` is True even when the caller sent no
+# `response_format` and the generation is therefore unconstrained -- so the
+# native conversation-create builds constraint machinery for a grammar nothing
+# will use, once per frame. `config_schema.json` on the pinned 0.16.0 has 17
+# keys (backend, cpu_thread_count, audio_backend, vision_backend, cache,
+# max_num_tokens, temperature, top_p, top_k, seed, gpu_decode_steps_per_sync,
+# speculative_decoding, thinking, thinking_budget, activation_data_type, plus
+# the `default`/`models` containers) and NONE of them reaches it; no request
+# field reaches it either, and both 0.15.0 and 0.16.1 construct it identically,
+# so a version move does not reach it. That is why install_litertlm_shim below
+# exists -- see it for the measurement and for why it is not a patch.
 #
 # Two settings, both MEASURED on an M4 Max (12P+4E/48G), litert-lm 0.16.0, Gemma
 # 4 E2B, delegate asserted GPU from the native log, 16 warm-ups before timing,
@@ -126,6 +141,86 @@ write_litertlm_config() {
   log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=true)"
 }
 
+# -- The one knob config.json cannot express -----------------------------------
+#
+# `litert-lm serve` builds `ConstrainedDecodingConfig(enable=True, provider=...)`
+# for EVERY request, at both openai_handler.py construction sites, with `enable`
+# hardcoded True -- so a request that carried no `response_format`, which is
+# every request the connector sends, still makes the native conversation-create
+# build constraint machinery for a grammar nothing will use. It is rebuilt per
+# request because the serve creates one conversation per request.
+#
+# MEASURED (M4 Max 16c/48G, litert-lm 0.16.0, Gemma 4 E2B, GPU/Metal, vision_backend
+# =gpu + speculative_decoding=true, operator/SOP field set, the two committed CCTV
+# fixtures, arms alternated sample-by-sample with the leading arm rotated each
+# round, load average 22-25):
+#
+#   in-process, one engine, n=10 pairs
+#     arm          create_conversation   send_message   total
+#     stock              0.717 s            0.560 s     1.275 s
+#     without it         0.001 s            0.603 s     0.604 s
+#     paired 2.194x (min 2.040, max 2.284); generations BYTE-IDENTICAL 10/10;
+#     prefill 493 / decode 58 on both arms
+#
+#   through two stock serves on two ports, interleaved, n=16 pairs
+#     1.309 s -> 0.592 s p50 (p95 3.664 -> 0.965), paired 2.42x (min 2.174, max
+#     4.657), 16/16 schema-valid and 16/16 byte-identical on both arms
+#
+#   bench-local-vlm.mjs --arms gpu-sop, 16 warm-ups, n=8, back to back
+#     1.32 s/frame (2,727 analyses/hour) -> 0.72 s/frame (5,000/hour), 8/8
+#     schema-valid and 83.3% field accuracy on BOTH arms
+#
+# A speed change, not an answer change -- the same bar speculative_decoding had
+# to clear. The RATIO is itself load-dependent (3.734x at 0.0% CPU idle vs 2.42x
+# at load 25, because the removed work is CPU-bound and the retained work is
+# GPU-bound), so quote the near-idle figure.
+#
+# We do NOT patch upstream. Two files are ADDED to the litert-lm tool venv's
+# site-packages: our shim, and a one-line .pth that imports it. Every upstream
+# file stays byte-identical, no launch definition changes on any platform (so
+# the model OTA's `launchctl kickstart com.trillboards.litertlm` still finds the
+# same label), and a serve started by hand gets the fix too. The shim is lazy --
+# it only registers an import hook, so `litert-lm list`/`--version`, which this
+# script runs as health checks, do not gain a dylib load. It fails OPEN: any
+# failure leaves stock behaviour and prints one line to the serve's own log, and
+# `TRILLBOARDS_LITERTLM_SHIM=0` disables it without uninstalling anything.
+#
+# Runs UNCONDITIONALLY, never inside the install/repair branch: `uv tool install
+# --force` rebuilds the venv and takes both files with it, and the healthy-CLI
+# path skips that branch entirely -- so gating this behind it is exactly how a
+# repaired-and-healthy machine would keep the slow serve while reporting fine.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || echo .)"
+find_litertlm_shim() {
+  for _c in "$SCRIPT_DIR/trillboards_litertlm_shim.py" \
+            "$SCRIPT_DIR/../litertlm/trillboards_litertlm_shim.py" \
+            "$SCRIPT_DIR/../../litertlm/trillboards_litertlm_shim.py"; do
+    [ -f "$_c" ] && { printf '%s\n' "$_c"; return 0; }
+  done
+  return 1
+}
+install_litertlm_shim() {
+  _shim="$(find_litertlm_shim)" || { log "WARN: litert-lm shim source not found; serve keeps stock (slower) conversation setup"; return 1; }
+  _tooldir="$("$UV" tool dir 2>/dev/null | tr -d '\r')"
+  [ -n "$_tooldir" ] || { log "WARN: could not resolve \`uv tool dir\`; skipping litert-lm shim"; return 1; }
+  _py="$_tooldir/litert-lm/bin/python"
+  [ -x "$_py" ] || { log "WARN: litert-lm venv python missing at $_py; skipping shim"; return 1; }
+  _site="$("$_py" -c 'import sysconfig;print(sysconfig.get_paths()["purelib"])' 2>/dev/null | tr -d '\r')"
+  [ -n "$_site" ] && [ -d "$_site" ] || { log "WARN: could not resolve litert-lm site-packages; skipping shim"; return 1; }
+  cp -f "$_shim" "$_site/trillboards_litertlm_shim.py" 2>/dev/null \
+    || { log "WARN: could not copy the litert-lm shim into $_site"; return 1; }
+  printf 'import trillboards_litertlm_shim\n' > "$_site/trillboards-litertlm-shim.pth" 2>/dev/null \
+    || { log "WARN: could not write the litert-lm shim .pth"; return 1; }
+  # Capability is established by ATTEMPTING THE OPERATION. Importing litert_lm
+  # here is the only thing that proves the wrap actually landed on the class the
+  # serve will use; a file that merely exists proves nothing.
+  if "$_py" -c 'import litert_lm,sys; sys.exit(0 if getattr(litert_lm.Engine.create_conversation,"_trillboards_litertlm_shim",False) else 1)' >>"$LOG" 2>&1; then
+    log "litert-lm fast-conversation shim active in $_site"
+    return 0
+  fi
+  log "WARN: litert-lm shim did not take effect; serve keeps stock (slower) conversation setup"
+  return 1
+}
+
 # One real image inference against the running serve. A 200 means the vision
 # encoder loaded AND ran on whatever backend config.json just asked for.
 # Deliberately NOT a /v1/models check: that answers 200 without ever building an
@@ -332,6 +427,8 @@ else
   log "litert-lm CLI repaired successfully at $("$LITERTLM" --version 2>/dev/null | awk '{print $NF}')"
 fi
 
+install_litertlm_shim || true
+
 # 3. Model - import the verified bundled file. Idempotent: skip entirely if
 #    litert-lm already has the model. Engineering runs may use HF only with the
 #    explicit TRILLBOARDS_ALLOW_MODEL_DOWNLOAD=1 opt-in.

package/deploy/systemd/setup-litertlm.sh

@@ -74,8 +74,23 @@ fail() { log "ERROR: $*"; exit 1; }
 # `litert-lm serve` takes NO flags for any of this: `serve` accepts only
 # --host/--port/--api/--cors-origin/--verbose. Every engine knob it honours comes
 # from `$LITERT_LM_DIR/config.json`, which `litert_lm_cli.config.get_model_config`
-# merges per model id. So this file IS the tuning surface, and writing it is the
-# whole change -- the serve stays stock.
+# merges per model id. So this file IS the tuning surface for everything it can
+# express, and the serve binary stays stock: we pass it no flags and we modify
+# none of its files.
+#
+# ONE knob is NOT expressible there, and it is the largest one left. Every
+# request the serve handles gets `ConstrainedDecodingConfig(enable=True, ...)`
+# built for it unconditionally -- `enable` is True even when the caller sent no
+# `response_format` and the generation is therefore unconstrained -- so the
+# native conversation-create builds constraint machinery for a grammar nothing
+# will use, once per frame. `config_schema.json` on the pinned 0.16.0 has 17
+# keys (backend, cpu_thread_count, audio_backend, vision_backend, cache,
+# max_num_tokens, temperature, top_p, top_k, seed, gpu_decode_steps_per_sync,
+# speculative_decoding, thinking, thinking_budget, activation_data_type, plus
+# the `default`/`models` containers) and NONE of them reaches it; no request
+# field reaches it either, and both 0.15.0 and 0.16.1 construct it identically,
+# so a version move does not reach it. That is why install_litertlm_shim below
+# exists -- see it for the measurement and for why it is not a patch.
 #
 # Two settings, both MEASURED on an M4 Max (12P+4E/48G), litert-lm 0.16.0, Gemma
 # 4 E2B, delegate asserted GPU from the native log, 16 warm-ups before timing,
@@ -134,6 +149,86 @@ write_litertlm_config() {
   log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=true)"
 }
 
+# -- The one knob config.json cannot express -----------------------------------
+#
+# `litert-lm serve` builds `ConstrainedDecodingConfig(enable=True, provider=...)`
+# for EVERY request, at both openai_handler.py construction sites, with `enable`
+# hardcoded True -- so a request that carried no `response_format`, which is
+# every request the connector sends, still makes the native conversation-create
+# build constraint machinery for a grammar nothing will use. It is rebuilt per
+# request because the serve creates one conversation per request.
+#
+# MEASURED (M4 Max 16c/48G, litert-lm 0.16.0, Gemma 4 E2B, GPU/Metal, vision_backend
+# =gpu + speculative_decoding=true, operator/SOP field set, the two committed CCTV
+# fixtures, arms alternated sample-by-sample with the leading arm rotated each
+# round, load average 22-25):
+#
+#   in-process, one engine, n=10 pairs
+#     arm          create_conversation   send_message   total
+#     stock              0.717 s            0.560 s     1.275 s
+#     without it         0.001 s            0.603 s     0.604 s
+#     paired 2.194x (min 2.040, max 2.284); generations BYTE-IDENTICAL 10/10;
+#     prefill 493 / decode 58 on both arms
+#
+#   through two stock serves on two ports, interleaved, n=16 pairs
+#     1.309 s -> 0.592 s p50 (p95 3.664 -> 0.965), paired 2.42x (min 2.174, max
+#     4.657), 16/16 schema-valid and 16/16 byte-identical on both arms
+#
+#   bench-local-vlm.mjs --arms gpu-sop, 16 warm-ups, n=8, back to back
+#     1.32 s/frame (2,727 analyses/hour) -> 0.72 s/frame (5,000/hour), 8/8
+#     schema-valid and 83.3% field accuracy on BOTH arms
+#
+# A speed change, not an answer change -- the same bar speculative_decoding had
+# to clear. The RATIO is itself load-dependent (3.734x at 0.0% CPU idle vs 2.42x
+# at load 25, because the removed work is CPU-bound and the retained work is
+# GPU-bound), so quote the near-idle figure.
+#
+# We do NOT patch upstream. Two files are ADDED to the litert-lm tool venv's
+# site-packages: our shim, and a one-line .pth that imports it. Every upstream
+# file stays byte-identical, no launch definition changes on any platform (the
+# systemd unit's ExecStart is untouched), and a serve started by hand gets the
+# fix too. The shim is lazy --
+# it only registers an import hook, so `litert-lm list`/`--version`, which this
+# script runs as health checks, do not gain a dylib load. It fails OPEN: any
+# failure leaves stock behaviour and prints one line to the serve's own log, and
+# `TRILLBOARDS_LITERTLM_SHIM=0` disables it without uninstalling anything.
+#
+# Runs UNCONDITIONALLY, never inside the install/repair branch: `uv tool install
+# --force` rebuilds the venv and takes both files with it, and the healthy-CLI
+# path skips that branch entirely -- so gating this behind it is exactly how a
+# repaired-and-healthy machine would keep the slow serve while reporting fine.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || echo .)"
+find_litertlm_shim() {
+  for _c in "$SCRIPT_DIR/trillboards_litertlm_shim.py" \
+            "$SCRIPT_DIR/../litertlm/trillboards_litertlm_shim.py" \
+            "$SCRIPT_DIR/../../litertlm/trillboards_litertlm_shim.py"; do
+    [ -f "$_c" ] && { printf '%s\n' "$_c"; return 0; }
+  done
+  return 1
+}
+install_litertlm_shim() {
+  _shim="$(find_litertlm_shim)" || { log "WARN: litert-lm shim source not found; serve keeps stock (slower) conversation setup"; return 1; }
+  _tooldir="$("$UV" tool dir 2>/dev/null | tr -d '\r')"
+  [ -n "$_tooldir" ] || { log "WARN: could not resolve \`uv tool dir\`; skipping litert-lm shim"; return 1; }
+  _py="$_tooldir/litert-lm/bin/python"
+  [ -x "$_py" ] || { log "WARN: litert-lm venv python missing at $_py; skipping shim"; return 1; }
+  _site="$("$_py" -c 'import sysconfig;print(sysconfig.get_paths()["purelib"])' 2>/dev/null | tr -d '\r')"
+  [ -n "$_site" ] && [ -d "$_site" ] || { log "WARN: could not resolve litert-lm site-packages; skipping shim"; return 1; }
+  cp -f "$_shim" "$_site/trillboards_litertlm_shim.py" 2>/dev/null \
+    || { log "WARN: could not copy the litert-lm shim into $_site"; return 1; }
+  printf 'import trillboards_litertlm_shim\n' > "$_site/trillboards-litertlm-shim.pth" 2>/dev/null \
+    || { log "WARN: could not write the litert-lm shim .pth"; return 1; }
+  # Capability is established by ATTEMPTING THE OPERATION. Importing litert_lm
+  # here is the only thing that proves the wrap actually landed on the class the
+  # serve will use; a file that merely exists proves nothing.
+  if "$_py" -c 'import litert_lm,sys; sys.exit(0 if getattr(litert_lm.Engine.create_conversation,"_trillboards_litertlm_shim",False) else 1)' >>"$LOG" 2>&1; then
+    log "litert-lm fast-conversation shim active in $_site"
+    return 0
+  fi
+  log "WARN: litert-lm shim did not take effect; serve keeps stock (slower) conversation setup"
+  return 1
+}
+
 # One real image inference against the running serve. A 200 means the vision
 # encoder loaded AND ran on whatever backend config.json just asked for.
 # Deliberately NOT a /v1/models check: that answers 200 without ever building an
@@ -226,6 +321,8 @@ else
   log "litert-lm CLI repaired at $("$LITERTLM" --version 2>/dev/null | awk '{print $NF}')"
 fi
 
+install_litertlm_shim || true
+
 # 3. Model. Idempotent: skip entirely if litert-lm already has it.
 HAVE_MODEL=0
 if "$LITERTLM" list 2>/dev/null | grep -q "$MODEL_ID"; then

package/deploy/windows/build-windows.ps1

@@ -222,6 +222,10 @@ Log "Customer setup window smoke test passed"
 # model path from the customer SFX and verifies local AI before pairing; the
 # serve task keeps `litert-lm serve :9379` alive for local-sense.
 Copy-Item -Force "$Here\setup-litertlm.ps1"         "$Payload\bin\setup-litertlm.ps1"
+# Staged NEXT TO setup-litertlm.ps1 because that is where Install-LitertlmShim
+# looks first. Copied into the litert-lm tool venv's site-packages at
+# provisioning time; nothing else reads it, and no upstream file is modified.
+Copy-Item -Force "$Here\..\litertlm\trillboards_litertlm_shim.py" "$Payload\bin\trillboards_litertlm_shim.py"
 Copy-Item -Force "$Here\register-litertlm-task.ps1" "$Payload\bin\register-litertlm-task.ps1"
 Copy-Item -Force "$Here\check-feeds.js"       "$Payload\bin\check-feeds.js"