@trillboards/edge-sdk 0.2.60 → 0.2.63

package/deploy/macos/test-setup-litertlm-reinstall.sh

@@ -36,8 +36,18 @@ fi
 exit 0
 EOF
 
+# Fake curl models BOTH endpoints the setup script uses:
+#   GET  /v1/models            -> the catalogue body `local_ai_healthy` greps
+#   POST /v1/chat/completions  -> the vision probe, which asks for `%{http_code}`
+# TRILLBOARDS_TEST_VISION_PROBE_CODE lets a test drive the GPU-vision fallback.
 cat >"$FAKE_BIN/curl" <<'EOF'
 #!/bin/bash
+for arg in "$@"; do
+  if [ "$arg" = "-w" ]; then
+    printf '%s' "${TRILLBOARDS_TEST_VISION_PROBE_CODE:-200}"
+    exit 0
+  fi
+done
 printf '{"data":[{"id":"gemma-4-E2B-it.litertlm"}]}\n'
 EOF
 
@@ -103,7 +113,55 @@ if ! OUTPUT="$(/bin/bash "$SETUP" 2>&1)"; then
 fi
 
 [ "$(cat "$TRILLBOARDS_TEST_STATE_DIR/bootstrap-attempts")" = "2" ]
+# A host whose GPU vision encoder answers keeps it: config.json must carry
+# vision_backend=gpu and must NOT have been rewritten by the fallback.
+grep -Fq '"vision_backend":"gpu"' "$LITERT_LM_DIR/config.json"
+grep -Fq '"speculative_decoding":true' "$LITERT_LM_DIR/config.json"
+grep -Fq 'local AI healthy with the GPU vision encoder' <<<"$OUTPUT"
 grep -Fq 'Bootstrap failed: 37: Operation already in progress' \
   "$HOME/Library/Logs/Trillboards/litertlm-setup.log"
 grep -Fq 'local AI is installed and healthy' <<<"$OUTPUT"
-printf 'macOS LiteRT-LM reinstall race regression passed\n'
+
+# -- Scenario 2: a host whose GPU vision encoder does NOT answer --------------
+#
+# This is the case the cascade exists for. Pinning the vision encoder to the GPU
+# is the one setting here that can make a host WORSE than it was, and there is no
+# demotion path for it: `recordGpuDemotion` re-posts the BARE model id, which
+# moves the MAIN backend and leaves vision_backend untouched. So a box whose GPU
+# will not bind must end this script serving on the CPU encoder, not dead.
+#
+# `local_ai_healthy` cannot catch it -- /v1/models answers 200 without ever
+# building an engine. Only a real image inference can, which is why the probe
+# POSTs one and why this test drives its status code.
+printf 'loaded\n' >"$TRILLBOARDS_TEST_STATE_DIR/service-state"
+printf '0\n' >"$TRILLBOARDS_TEST_STATE_DIR/bootstrap-attempts"
+export TRILLBOARDS_TEST_VISION_PROBE_CODE=500
+
+if ! OUTPUT2="$(/bin/bash "$SETUP" 2>&1)"; then
+  printf '%s\n' "$OUTPUT2" >&2
+  printf 'FAIL: setup exited non-zero when the GPU vision probe failed; it must fall back, not fail\n' >&2
+  exit 1
+fi
+
+grep -Fq 'falling back to the CPU encoder' <<<"$OUTPUT2"
+grep -Fq 'local AI healthy on the CPU vision encoder' <<<"$OUTPUT2"
+grep -Fq 'local AI is installed and healthy' <<<"$OUTPUT2"
+
+# The fallback must REMOVE vision_backend and KEEP speculative_decoding: the
+# latter is delegate independent and byte-identical to greedy, so it is still
+# correct on a CPU-vision host and is the only speedup that host keeps.
+if grep -Fq 'vision_backend' "$LITERT_LM_DIR/config.json"; then
+  printf 'FAIL: fallback left vision_backend in config.json\n' >&2
+  cat "$LITERT_LM_DIR/config.json" >&2
+  exit 1
+fi
+grep -Fq '"speculative_decoding":true' "$LITERT_LM_DIR/config.json"
+
+# 2 bootstraps for the first start (attempt 1 hits the 37 race), + 1 for the
+# restart the fallback performs.
+[ "$(cat "$TRILLBOARDS_TEST_STATE_DIR/bootstrap-attempts")" = "3" ]
+
+unset TRILLBOARDS_TEST_VISION_PROBE_CODE
+
+printf 'macOS LiteRT-LM reinstall race + GPU-vision fallback regressions passed\n'
+

package/deploy/systemd/setup-litertlm.sh

@@ -69,6 +69,182 @@ log() {
   printf '[trillboards setup-litertlm] %s\n' "$*"
 }
 fail() { log "ERROR: $*"; exit 1; }
+# -- Local-inference tuning (config.json) ------------------------------------
+#
+# `litert-lm serve` takes NO flags for any of this: `serve` accepts only
+# --host/--port/--api/--cors-origin/--verbose. Every engine knob it honours comes
+# from `$LITERT_LM_DIR/config.json`, which `litert_lm_cli.config.get_model_config`
+# merges per model id. So this file IS the tuning surface for everything it can
+# express, and the serve binary stays stock: we pass it no flags and we modify
+# none of its files.
+#
+# ONE knob is NOT expressible there, and it is the largest one left. Every
+# request the serve handles gets `ConstrainedDecodingConfig(enable=True, ...)`
+# built for it unconditionally -- `enable` is True even when the caller sent no
+# `response_format` and the generation is therefore unconstrained -- so the
+# native conversation-create builds constraint machinery for a grammar nothing
+# will use, once per frame. `config_schema.json` on the pinned 0.16.0 has 17
+# keys (backend, cpu_thread_count, audio_backend, vision_backend, cache,
+# max_num_tokens, temperature, top_p, top_k, seed, gpu_decode_steps_per_sync,
+# speculative_decoding, thinking, thinking_budget, activation_data_type, plus
+# the `default`/`models` containers) and NONE of them reaches it; no request
+# field reaches it either, and both 0.15.0 and 0.16.1 construct it identically,
+# so a version move does not reach it. That is why install_litertlm_shim below
+# exists -- see it for the measurement and for why it is not a patch.
+#
+# Two settings, both MEASURED on an M4 Max (12P+4E/48G), litert-lm 0.16.0, Gemma
+# 4 E2B, delegate asserted GPU from the native log, 16 warm-ups before timing,
+# arms interleaved sample-by-sample, operator/SOP field set + the two committed
+# CCTV fixtures:
+#
+#   vision_backend=gpu    The vision encoder ran on the CPU because NOTHING ASKED
+#                         IT NOT TO -- `Engine(vision_backend=...)` defaults to
+#                         None and the 0.13.1 HTTP shim hardcoded Backend.CPU()
+#                         (upstream b/515805503). It is not a hardware limit and
+#                         not a model constraint; GPU vision is Google's own
+#                         reference config and CPU is their FALLBACK tier.
+#                         Image encode 0.804 s -> 0.175 s p50, i.e. 76% of a SOP
+#                         frame removed.
+#   speculative_decoding  1.433x on the decode-heavy audience schema, and the
+#                         generations are BYTE-IDENTICAL to greedy (12/12 same
+#                         text), which is the property that makes it safe: it is
+#                         a speed change, not an answer change.
+#
+# Together, in-process: 1.054 s -> 0.385 s per SOP frame (2.765x paired, min
+# 2.560 / max 2.919 over 16 rounds); 3,416 -> 9,295 analyses/hour. Through this
+# stock serve over HTTP: 1752.5 ms -> 1005.8 ms p50 (1.742x), 12/12 schema-valid
+# on BOTH arms. Accuracy is not traded for it: people_present 16/16 on both arms,
+# person_count 8/16 on both, and where they differ the GPU encoder is CLOSER to
+# the hand label (4 vs 3 against a truth of 5).
+#
+# WHY THE VERIFY BELOW EXISTS. Pinning the vision encoder to the GPU is the one
+# setting here that can make a host WORSE than it is today: a box whose GPU will
+# not bind fails engine init, and unlike the main backend there is no demotion
+# path for it -- `LiteRtVlmProvider.recordGpuDemotion` re-posts the BARE model
+# id, which changes the main backend and leaves this config untouched. So a
+# blind write would brick sensing on exactly the weak hosts that need it most.
+# We therefore do what Google's own cascade does: try GPU vision, prove it with
+# one real image inference, and fall back to the CPU default if it does not
+# answer.
+CONFIG_JSON="$LITERT_DIR/config.json"
+
+# A 223-byte 32x32 grey JPEG. Small enough to inline, real enough that decoding
+# it exercises the vision encoder we are trying to prove.
+LITERTLM_PROBE_JPEG_B64="/9j/4AAQSkZJRgABAgAAAQABAAD//gAPTGF2YzYzLjEuMTAxAP/bAEMACD4+ST5JVVVVVVVVZF1kaGhoZGRkZGhoaHBwcIODg3BwcGhocHB8fIODj5OPh4eDh5OTm5uburqystnZ4P/////EAEoAAQAAAAAAAAAAAAAAAAAAAAABAQAAAAAAAAAAAAAAAAAAAAAQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCAAgACADASIAAhEAAxEA/9oADAMBAAIRAxEAPwAAAAAAAAD/2Q=="
+
+# Write config.json. $1 = "gpu" pins the vision encoder; empty leaves it on the
+# CPU default. speculative_decoding is written EITHER WAY -- it is delegate
+# independent and byte-identical, so it survives the fallback.
+write_litertlm_config() {
+  _vision="$1"
+  mkdir -p "$LITERT_DIR" 2>/dev/null || true
+  if [ -n "$_vision" ]; then
+    printf '{"models":{"%s":{"vision_backend":"%s","speculative_decoding":true}}}\n' \
+      "$MODEL_ID" "$_vision" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
+  else
+    printf '{"models":{"%s":{"speculative_decoding":true}}}\n' \
+      "$MODEL_ID" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
+  fi
+  mv -f "$CONFIG_JSON.tmp" "$CONFIG_JSON" 2>/dev/null || return 1
+  log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=true)"
+}
+
+# -- The one knob config.json cannot express -----------------------------------
+#
+# `litert-lm serve` builds `ConstrainedDecodingConfig(enable=True, provider=...)`
+# for EVERY request, at both openai_handler.py construction sites, with `enable`
+# hardcoded True -- so a request that carried no `response_format`, which is
+# every request the connector sends, still makes the native conversation-create
+# build constraint machinery for a grammar nothing will use. It is rebuilt per
+# request because the serve creates one conversation per request.
+#
+# MEASURED (M4 Max 16c/48G, litert-lm 0.16.0, Gemma 4 E2B, GPU/Metal, vision_backend
+# =gpu + speculative_decoding=true, operator/SOP field set, the two committed CCTV
+# fixtures, arms alternated sample-by-sample with the leading arm rotated each
+# round, load average 22-25):
+#
+#   in-process, one engine, n=10 pairs
+#     arm          create_conversation   send_message   total
+#     stock              0.717 s            0.560 s     1.275 s
+#     without it         0.001 s            0.603 s     0.604 s
+#     paired 2.194x (min 2.040, max 2.284); generations BYTE-IDENTICAL 10/10;
+#     prefill 493 / decode 58 on both arms
+#
+#   through two stock serves on two ports, interleaved, n=16 pairs
+#     1.309 s -> 0.592 s p50 (p95 3.664 -> 0.965), paired 2.42x (min 2.174, max
+#     4.657), 16/16 schema-valid and 16/16 byte-identical on both arms
+#
+#   bench-local-vlm.mjs --arms gpu-sop, 16 warm-ups, n=8, back to back
+#     1.32 s/frame (2,727 analyses/hour) -> 0.72 s/frame (5,000/hour), 8/8
+#     schema-valid and 83.3% field accuracy on BOTH arms
+#
+# A speed change, not an answer change -- the same bar speculative_decoding had
+# to clear. The RATIO is itself load-dependent (3.734x at 0.0% CPU idle vs 2.42x
+# at load 25, because the removed work is CPU-bound and the retained work is
+# GPU-bound), so quote the near-idle figure.
+#
+# We do NOT patch upstream. Two files are ADDED to the litert-lm tool venv's
+# site-packages: our shim, and a one-line .pth that imports it. Every upstream
+# file stays byte-identical, no launch definition changes on any platform (the
+# systemd unit's ExecStart is untouched), and a serve started by hand gets the
+# fix too. The shim is lazy --
+# it only registers an import hook, so `litert-lm list`/`--version`, which this
+# script runs as health checks, do not gain a dylib load. It fails OPEN: any
+# failure leaves stock behaviour and prints one line to the serve's own log, and
+# `TRILLBOARDS_LITERTLM_SHIM=0` disables it without uninstalling anything.
+#
+# Runs UNCONDITIONALLY, never inside the install/repair branch: `uv tool install
+# --force` rebuilds the venv and takes both files with it, and the healthy-CLI
+# path skips that branch entirely -- so gating this behind it is exactly how a
+# repaired-and-healthy machine would keep the slow serve while reporting fine.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || echo .)"
+find_litertlm_shim() {
+  for _c in "$SCRIPT_DIR/trillboards_litertlm_shim.py" \
+            "$SCRIPT_DIR/../litertlm/trillboards_litertlm_shim.py" \
+            "$SCRIPT_DIR/../../litertlm/trillboards_litertlm_shim.py"; do
+    [ -f "$_c" ] && { printf '%s\n' "$_c"; return 0; }
+  done
+  return 1
+}
+install_litertlm_shim() {
+  _shim="$(find_litertlm_shim)" || { log "WARN: litert-lm shim source not found; serve keeps stock (slower) conversation setup"; return 1; }
+  _tooldir="$("$UV" tool dir 2>/dev/null | tr -d '\r')"
+  [ -n "$_tooldir" ] || { log "WARN: could not resolve \`uv tool dir\`; skipping litert-lm shim"; return 1; }
+  _py="$_tooldir/litert-lm/bin/python"
+  [ -x "$_py" ] || { log "WARN: litert-lm venv python missing at $_py; skipping shim"; return 1; }
+  _site="$("$_py" -c 'import sysconfig;print(sysconfig.get_paths()["purelib"])' 2>/dev/null | tr -d '\r')"
+  [ -n "$_site" ] && [ -d "$_site" ] || { log "WARN: could not resolve litert-lm site-packages; skipping shim"; return 1; }
+  cp -f "$_shim" "$_site/trillboards_litertlm_shim.py" 2>/dev/null \
+    || { log "WARN: could not copy the litert-lm shim into $_site"; return 1; }
+  printf 'import trillboards_litertlm_shim\n' > "$_site/trillboards-litertlm-shim.pth" 2>/dev/null \
+    || { log "WARN: could not write the litert-lm shim .pth"; return 1; }
+  # Capability is established by ATTEMPTING THE OPERATION. Importing litert_lm
+  # here is the only thing that proves the wrap actually landed on the class the
+  # serve will use; a file that merely exists proves nothing.
+  if "$_py" -c 'import litert_lm,sys; sys.exit(0 if getattr(litert_lm.Engine.create_conversation,"_trillboards_litertlm_shim",False) else 1)' >>"$LOG" 2>&1; then
+    log "litert-lm fast-conversation shim active in $_site"
+    return 0
+  fi
+  log "WARN: litert-lm shim did not take effect; serve keeps stock (slower) conversation setup"
+  return 1
+}
+
+# One real image inference against the running serve. A 200 means the vision
+# encoder loaded AND ran on whatever backend config.json just asked for.
+# Deliberately NOT a /v1/models check: that answers 200 without ever building an
+# engine, so it cannot tell a working GPU encoder from one that fails on first
+# use.
+verify_litertlm_vision() {
+  _body="$LITERT_DIR/.vision-probe.json"
+  printf '{"model":"%s,gpu","messages":[{"role":"user","content":[{"type":"text","text":"Reply with the single word: ok"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,%s"}}]}],"max_completion_tokens":8,"temperature":0}' \
+    "$MODEL_ID" "$LITERTLM_PROBE_JPEG_B64" > "$_body" 2>/dev/null || return 1
+  _code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 180 \
+    -X POST "http://127.0.0.1:$SERVE_PORT/v1/chat/completions" \
+    -H 'Content-Type: application/json' --data-binary @"$_body" 2>/dev/null)
+  rm -f "$_body" 2>/dev/null || true
+  [ "$_code" = "200" ]
+}
+
 local_ai_healthy() {
   curl -fsS --max-time 3 "http://127.0.0.1:$SERVE_PORT/v1/models" 2>/dev/null \
     | grep -Fq "$MODEL_ID"
@@ -145,6 +321,8 @@ else
   log "litert-lm CLI repaired at $("$LITERTLM" --version 2>/dev/null | awk '{print $NF}')"
 fi
 
+install_litertlm_shim || true
+
 # 3. Model. Idempotent: skip entirely if litert-lm already has it.
 HAVE_MODEL=0
 if "$LITERTLM" list 2>/dev/null | grep -q "$MODEL_ID"; then
@@ -172,25 +350,55 @@ fi
 
 # 4. Serve unit. Installed by install.sh (which has sudo); here we only start it
 # when we can, so a re-run from the service user still recovers a stopped serve.
-if command -v systemctl >/dev/null 2>&1; then
+restart_litertlm_serve() {
+  command -v systemctl >/dev/null 2>&1 || return 0
   if systemctl list-unit-files "$SERVE_UNIT.service" >/dev/null 2>&1; then
     systemctl restart "$SERVE_UNIT" >>"$LOG" 2>&1 \
       || sudo -n systemctl restart "$SERVE_UNIT" >>"$LOG" 2>&1 \
-      || log "WARN: could not restart $SERVE_UNIT — start it with: sudo systemctl restart $SERVE_UNIT"
+      || log "WARN: could not restart $SERVE_UNIT - start it with: sudo systemctl restart $SERVE_UNIT"
   else
     log "WARN: $SERVE_UNIT.service is not installed yet (install.sh installs it)"
   fi
-fi
+}
+
+wait_for_litertlm_health() {
+  log "waiting for local AI health check on :$SERVE_PORT"
+  for _attempt in $(seq 1 60); do
+    if local_ai_healthy; then return 0; fi
+    sleep 2
+  done
+  return 1
+}
+
+# Ask for GPU vision BEFORE the restart so the engine builds with it. The engine
+# is constructed lazily on the first inference, so config.json must be on disk
+# before that request, not merely before the next boot.
+write_litertlm_config "gpu" || log "WARN: could not write $CONFIG_JSON - the serve will use its defaults"
+
+restart_litertlm_serve
 
-log "waiting for local AI health check on :$SERVE_PORT"
 HEALTHY=0
-for _attempt in $(seq 1 60); do
-  if local_ai_healthy; then HEALTHY=1; break; fi
-  sleep 2
-done
+if wait_for_litertlm_health; then HEALTHY=1; fi
+
 if [ "$HEALTHY" -eq 1 ]; then
-  log "local AI is installed and healthy — the connector will run ON-DEVICE"
-else
+  # The cascade. `local_ai_healthy` only proves /v1/models answers, which it does
+  # without ever building an engine - so it CANNOT see a GPU vision encoder that
+  # fails on first use. One real image inference is the only thing that can.
+  if verify_litertlm_vision; then
+    log "local AI healthy with the GPU vision encoder - the connector will run ON-DEVICE"
+  else
+    log "WARN: GPU vision encoder did not answer a probe inference - falling back to the CPU encoder"
+    write_litertlm_config "" || log "WARN: could not rewrite $CONFIG_JSON"
+    restart_litertlm_serve
+    if wait_for_litertlm_health; then
+      log "local AI healthy on the CPU vision encoder (slower, but serving)"
+    else
+      HEALTHY=0
+    fi
+  fi
+fi
+
+if [ "$HEALTHY" -ne 1 ]; then
   # NOT fatal. auto-sense waits and beacons, so an install that gets this far
   # with the model on disk recovers by itself once the serve comes up.
   log "WARN: Gemma is installed but the serve did not answer yet; auto-sense will"

package/deploy/windows/build-windows.ps1

@@ -222,6 +222,10 @@ Log "Customer setup window smoke test passed"
 # model path from the customer SFX and verifies local AI before pairing; the
 # serve task keeps `litert-lm serve :9379` alive for local-sense.
 Copy-Item -Force "$Here\setup-litertlm.ps1"         "$Payload\bin\setup-litertlm.ps1"
+# Staged NEXT TO setup-litertlm.ps1 because that is where Install-LitertlmShim
+# looks first. Copied into the litert-lm tool venv's site-packages at
+# provisioning time; nothing else reads it, and no upstream file is modified.
+Copy-Item -Force "$Here\..\litertlm\trillboards_litertlm_shim.py" "$Payload\bin\trillboards_litertlm_shim.py"
 Copy-Item -Force "$Here\register-litertlm-task.ps1" "$Payload\bin\register-litertlm-task.ps1"
 Copy-Item -Force "$Here\check-feeds.js"       "$Payload\bin\check-feeds.js"
 

package/deploy/windows/register-litertlm-task.ps1

@@ -26,6 +26,28 @@ $TaskName = "TrillboardsLiteRtLm"
 $UserId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
 # litert-lm serve --host 127.0.0.1 --port <port>  (loopback only -- the connector
 # is local; never expose the model server off-box).
+#
+# THE SERVE'S OWN LOG IS DISCARDED ON WINDOWS, AND ONLY ON WINDOWS. A Scheduled
+# Task has no stdout, so everything `litert-lm serve` prints goes nowhere. The
+# other two platforms keep it: the macOS LaunchAgent sets StandardOutPath /
+# StandardErrorPath (com.trillboards.litertlm.plist), and the unit sets
+# StandardOutput=journal (trillboards-litertlm.service). `litert-lm serve` has no
+# --log-file of its own -- only --verbose -- so redirection is the only route.
+#
+# This is not cosmetic. On 2026-08-19 windows-latest could not complete a single
+# frame on the shipped config (run 32227929611: both provisioner probes died with
+# connection-level errors, then two 12-minute request timeouts), and the reason is
+# STILL unknown for exactly this reason -- the one process that knows why wrote
+# its explanation to a handle nobody was holding.
+#
+# DO NOT "fix" this by wrapping the action in cmd.exe /c "... >> log 2>&1". The
+# stop path below filters Win32_Process on Name = python.exe|litert-lm.exe|
+# pythonw.exe, so a cmd.exe parent is invisible to it and would be orphaned on
+# every re-register, and Task Scheduler's restart-on-failure would then be
+# watching the wrapper instead of the server. Capturing this needs either an
+# upstream --log-file or a purpose-built launcher that keeps the image name, and
+# it must be proved against the S4U/boot-trigger ladder (the 0.2.17 incident)
+# before it ships.
 $argument = 'serve --host 127.0.0.1 --port ' + $Port
 $Action  = New-ScheduledTaskAction  -Execute $LitertLmPath -Argument $argument
 

package/deploy/windows/setup-litertlm.ps1

@@ -133,6 +133,105 @@ function Test-ServeHealthy {
 # that sentence stays gone rather than merely de-prioritised.
 #
 # Never throws: a diagnostic must not be able to fail an install.
+# -- Local-inference tuning (config.json) ------------------------------------
+#
+# `litert-lm serve` takes NO flags for any of this: serve accepts only
+# --host/--port/--api/--cors-origin/--verbose. Every engine knob it honours comes
+# from $LITERT_LM_DIR\config.json, merged per model id by
+# litert_lm_cli.config.get_model_config. So this file IS the tuning surface for
+# everything it can express, and the serve binary stays stock: we pass it no
+# flags and we modify none of its files.
+#
+# ONE knob is NOT expressible there, and it is the largest one left. Every
+# request the serve handles gets ConstrainedDecodingConfig(enable=True, ...)
+# built for it unconditionally -- enable is True even when the caller sent no
+# response_format and the generation is therefore unconstrained -- so the native
+# conversation-create builds constraint machinery for a grammar nothing will
+# use, once per frame. config_schema.json on the pinned 0.16.0 has 17 keys and
+# NONE reaches it; no request field reaches it either, and 0.15.0 and 0.16.1
+# construct it identically, so a version move does not reach it. That is why
+# Install-LitertlmShim below exists -- see it for the measurement.
+#
+# MEASURED (M4 Max 12P+4E/48G, litert-lm 0.16.0, Gemma 4 E2B, delegate asserted
+# GPU from the native log, 16 warm-ups, arms interleaved, operator/SOP field set,
+# the two committed CCTV fixtures): image encode 0.804 s -> 0.175 s p50 and the
+# whole SOP frame 1.054 s -> 0.385 s (2.765x paired); through the stock serve
+# over HTTP 1752.5 ms -> 1005.8 ms p50 (1.742x), 12/12 schema-valid on both arms.
+# The vision encoder was on the CPU because NOTHING ASKED IT NOT TO -- GPU vision
+# is Google's reference config and CPU is their documented fallback tier.
+# speculative_decoding is written too: 1.433x on the decode-heavy schema with
+# BYTE-IDENTICAL generations, so it is a speed change and not an answer change.
+$script:LitertlmProbeJpegB64 = "/9j/4AAQSkZJRgABAgAAAQABAAD//gAPTGF2YzYzLjEuMTAxAP/bAEMACD4+ST5JVVVVVVVVZF1kaGhoZGRkZGhoaHBwcIODg3BwcGhocHB8fIODj5OPh4eDh5OTm5uburqystnZ4P/////EAEoAAQAAAAAAAAAAAAAAAAAAAAABAQAAAAAAAAAAAAAAAAAAAAAQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCAAgACADASIAAhEAAxEA/9oADAMBAAIRAxEAPwAAAAAAAAD/2Q=="
+
+function Get-LitertlmConfigPath { Join-Path $LitertDir "config.json" }
+
+# $VisionBackend = "gpu" pins the vision encoder; "" leaves it on the CPU
+# default. speculative_decoding is written EITHER WAY -- it is delegate
+# independent, so it survives the fallback.
+function Write-LitertlmConfig {
+  param([string]$VisionBackend = "")
+  try {
+    if (-not (Test-Path -LiteralPath $LitertDir)) {
+      New-Item -ItemType Directory -Force -Path $LitertDir | Out-Null
+    }
+    $model = [ordered]@{}
+    if ($VisionBackend) { $model["vision_backend"] = $VisionBackend }
+    $model["speculative_decoding"] = $true
+    $cfg = [ordered]@{ models = [ordered]@{ $ModelId = $model } }
+    $path = Get-LitertlmConfigPath
+    $tmp = "$path.tmp"
+    # UTF-8 WITHOUT a BOM, written explicitly rather than via -Encoding.
+    #
+    # `Set-Content -Encoding UTF8` emits a BOM on Windows PowerShell 5.1 (it is
+    # BOM-less only on PowerShell 7+), and this script runs under whichever host
+    # the installer happens to invoke. litert-lm parses this file with a plain
+    # `json.load`, which REFUSES a BOM:
+    #     Failed to parse config.json: Unexpected UTF-8 BOM (decode using utf-8-sig)
+    # That is a 500 on EVERY inference, not a degraded mode -- and because
+    # /v1/models answers 200 without building an engine, the install still looked
+    # healthy. Measured on windows-latest 2026-08-19, run 32223243603: 0 frames
+    # out of a 3-arm matrix, every arm HTTP 500. The macOS and Linux
+    # provisioners write this file with `printf` and were never affected.
+    [System.IO.File]::WriteAllText(
+      $tmp,
+      ($cfg | ConvertTo-Json -Depth 6),
+      (New-Object System.Text.UTF8Encoding($false)))
+    Move-Item -LiteralPath $tmp -Destination $path -Force
+    $label = if ($VisionBackend) { $VisionBackend } else { "cpu-default" }
+    Log "wrote $path (vision_backend=$label, speculative_decoding=true)"
+    return $true
+  } catch {
+    Log "WARN: could not write litert-lm config.json: $_"
+    return $false
+  }
+}
+
+# One real image inference. A 200 means the vision encoder loaded AND ran on the
+# backend config.json just asked for. Deliberately NOT Test-ServeHealthy:
+# /v1/models answers without ever building an engine, so it cannot tell a working
+# GPU encoder from one that fails on first use.
+function Test-LitertlmVision {
+  try {
+    $dataUri = "data:image/jpeg;base64," + $script:LitertlmProbeJpegB64
+    $body = @{
+      model = "$ModelId,gpu"
+      messages = @(@{ role = "user"; content = @(
+        @{ type = "text"; text = "Reply with the single word: ok" },
+        @{ type = "image_url"; image_url = @{ url = $dataUri } }
+      )})
+      max_completion_tokens = 8
+      temperature = 0
+    } | ConvertTo-Json -Depth 8
+    $resp = Invoke-WebRequest -Uri "http://127.0.0.1:$Port/v1/chat/completions" `
+      -Method POST -ContentType "application/json" -Body $body `
+      -TimeoutSec 180 -UseBasicParsing
+    return ($resp.StatusCode -eq 200)
+  } catch {
+    Log "vision probe did not answer: $_"
+    return $false
+  }
+}
+
 function Get-AdvertisedModelIds {
   try {
     $response = Invoke-WebRequest -Uri "http://127.0.0.1:$Port/v1/models" -UseBasicParsing -TimeoutSec 5
@@ -376,6 +475,90 @@ if ((Test-LitertLmCli) -and ($installedLitertLmVersion -eq $LitertLmVersion)) {
   Log "litert-lm CLI repaired successfully at $(Get-LitertLmVersion)"
 }
 
+# -- The one knob config.json cannot express -----------------------------------
+#
+# `litert-lm serve` builds ConstrainedDecodingConfig(enable=True, provider=...)
+# for EVERY request, at both openai_handler.py construction sites, with enable
+# hardcoded True -- so a request that carried no response_format, which is every
+# request the connector sends, still makes the native conversation-create build
+# constraint machinery for a grammar nothing will use. It is rebuilt per request
+# because the serve creates one conversation per request.
+#
+# MEASURED (M4 Max 16c/48G, litert-lm 0.16.0, Gemma 4 E2B, GPU/Metal, vision_backend
+# =gpu + speculative_decoding=true, operator/SOP field set, the two committed CCTV
+# fixtures, arms alternated sample-by-sample with the leading arm rotated each
+# round, load average 22-25):
+#
+#   in-process, one engine, n=10 pairs
+#     arm          create_conversation   send_message   total
+#     stock              0.717 s            0.560 s     1.275 s
+#     without it         0.001 s            0.603 s     0.604 s
+#     paired 2.194x (min 2.040, max 2.284); generations BYTE-IDENTICAL 10/10;
+#     prefill 493 / decode 58 on both arms
+#
+#   through two stock serves on two ports, interleaved, n=16 pairs
+#     1.309 s -> 0.592 s p50 (p95 3.664 -> 0.965), paired 2.42x (min 2.174, max
+#     4.657), 16/16 schema-valid and 16/16 byte-identical on both arms
+#
+#   bench-local-vlm.mjs --arms gpu-sop, 16 warm-ups, n=8, back to back
+#     1.32 s/frame (2,727 analyses/hour) -> 0.72 s/frame (5,000/hour), 8/8
+#     schema-valid and 83.3% field accuracy on BOTH arms
+#
+# A speed change, not an answer change -- the same bar speculative_decoding had
+# to clear. The RATIO is itself load-dependent (3.734x at 0.0% CPU idle vs 2.42x
+# at load 25, because the removed work is CPU-bound and the retained work is
+# GPU-bound), so quote the near-idle figure.
+#
+# We do NOT patch upstream. Two files are ADDED to the litert-lm tool venv's
+# site-packages: our shim, and a one-line .pth that imports it. Every upstream
+# file stays byte-identical, the Scheduled Task's action is untouched (so the
+# model OTA's `schtasks /End; /Run /TN TrillboardsLiteRtLm` still finds the same
+# task), and a serve started by hand gets the fix too. It fails OPEN: any
+# failure leaves stock behaviour and prints one line to the serve's own log, and
+# TRILLBOARDS_LITERTLM_SHIM=0 disables it without uninstalling anything.
+#
+# Runs UNCONDITIONALLY, never inside the install/repair branch: `uv tool install
+# --force` rebuilds the venv and takes both files with it, while Test-LitertLmCli
+# short-circuits that branch on a healthy CLI -- gating this behind it is exactly
+# how a repaired-and-healthy machine would keep the slow serve and report fine.
+# Never throws: this is a speed fix, and it must not be able to fail an install.
+function Install-LitertlmShim {
+  param([string]$UvPath)
+  try {
+    $shim = $null
+    foreach ($c in @(
+      (Join-Path $PSScriptRoot "trillboards_litertlm_shim.py"),
+      (Join-Path $PSScriptRoot "..\litertlm\trillboards_litertlm_shim.py")
+    )) { if (Test-Path -LiteralPath $c -PathType Leaf) { $shim = $c; break } }
+    if (-not $shim) { Log "WARN: litert-lm shim source not found; serve keeps stock (slower) conversation setup"; return $false }
+
+    $toolDir = (& $UvPath tool dir 2>$null | Select-Object -First 1)
+    if ([string]::IsNullOrWhiteSpace($toolDir)) { Log "WARN: could not resolve 'uv tool dir'; skipping litert-lm shim"; return $false }
+    $py = Join-Path $toolDir.Trim() "litert-lm\Scripts\python.exe"
+    if (-not (Test-Path -LiteralPath $py -PathType Leaf)) { Log "WARN: litert-lm venv python missing at $py; skipping shim"; return $false }
+
+    $site = (& $py -c "import sysconfig;print(sysconfig.get_paths()['purelib'])" 2>$null | Select-Object -First 1)
+    if ([string]::IsNullOrWhiteSpace($site) -or -not (Test-Path -LiteralPath $site.Trim())) {
+      Log "WARN: could not resolve litert-lm site-packages; skipping shim"; return $false
+    }
+    $site = $site.Trim()
+    Copy-Item -LiteralPath $shim -Destination (Join-Path $site "trillboards_litertlm_shim.py") -Force
+    Set-Content -LiteralPath (Join-Path $site "trillboards-litertlm-shim.pth") -Value "import trillboards_litertlm_shim" -Encoding ASCII
+
+    # Capability is established by ATTEMPTING THE OPERATION. Importing litert_lm
+    # here is the only thing that proves the wrap landed on the class the serve
+    # will use; a file that merely exists proves nothing.
+    & $py -c "import litert_lm,sys; sys.exit(0 if getattr(litert_lm.Engine.create_conversation,'_trillboards_litertlm_shim',False) else 1)" *>> $LogPath
+    if ($LASTEXITCODE -eq 0) { Log "litert-lm fast-conversation shim active in $site"; return $true }
+    Log "WARN: litert-lm shim did not take effect; serve keeps stock (slower) conversation setup"
+    return $false
+  } catch {
+    Log "WARN: litert-lm shim install failed: $_"
+    return $false
+  }
+}
+Install-LitertlmShim -UvPath $uvCmd.Source | Out-Null
+
 $bundledModelVerified = $false
 if (-not [string]::IsNullOrWhiteSpace($BundledModelPath)) {
   if (-not (Test-Path -LiteralPath $BundledModelPath -PathType Leaf)) {
@@ -475,6 +658,10 @@ if (-not (Test-Path -LiteralPath $registration)) {
   exit 1
 }
 
+# Ask for GPU vision BEFORE the serve starts: the engine is built lazily on the
+# first inference, so config.json must be on disk before that request.
+Write-LitertlmConfig -VisionBackend "gpu" | Out-Null
+
 Log "registering litert-lm serve Scheduled Task on :$Port"
 & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $registration -LitertLmPath $litertlmPath -Port $Port *>> $LogPath
 if ($LASTEXITCODE -ne 0) {
@@ -495,6 +682,47 @@ if (-not $healthy) {
   exit 1
 }
 
+# The GPU-vision cascade. Health only proves /v1/models answers, and that route
+# never builds an engine -- so it cannot see a GPU vision encoder that fails on
+# first USE. If the probe inference does not answer we fall back to the CPU
+# encoder rather than leave this host unable to sense at all.
+if (-not (Test-LitertlmVision)) {
+  Log "WARN: GPU vision encoder did not answer a probe inference; falling back to the CPU encoder"
+  Write-LitertlmConfig | Out-Null
+  & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $registration -LitertLmPath $litertlmPath -Port $Port *>> $LogPath
+  $healthy = $false
+  for ($attempt = 0; $attempt -lt 60 -and -not $healthy; $attempt++) {
+    $healthy = Test-ServeHealthy
+    if (-not $healthy) { Start-Sleep -Seconds 2 }
+  }
+  if (-not $healthy) {
+    Fail-LocalAi "Gemma is installed, but the local AI service did not respond. Restart Windows, then retry Local AI."
+    Stop-SetupLock
+    exit 1
+  }
+  # RE-PROBE, with the same probe that sent us here.
+  #
+  # Test-ServeHealthy above only proves /v1/models answers, and this file's own
+  # comment six lines up says that route never builds an engine. So before this
+  # check existed, a CPU fallback that ALSO could not infer logged "healthy on
+  # the CPU vision encoder (slower, but serving)" and exited 0 on a machine where
+  # every frame 500s. That is exactly how the UTF-8 BOM above shipped as a GREEN
+  # install: the cascade detected the failure, downgraded a tier, and then
+  # verified the downgrade with a check that cannot fail for this reason.
+  #
+  # A CPU fallback that cannot infer is not a slower tier. It is a broken
+  # install, and the operator has to hear so at setup time rather than discover
+  # it as silence from a camera weeks later.
+  if (-not (Test-LitertlmVision)) {
+    Fail-LocalAi "Gemma is installed and the local AI service is answering, but it could not analyze a test image on either the GPU or the CPU vision encoder. See $LogPath."
+    Stop-SetupLock
+    exit 1
+  }
+  Log "local AI healthy on the CPU vision encoder (slower, but serving)"
+} else {
+  Log "local AI healthy with the GPU vision encoder"
+}
+
 Complete-Setup
 Stop-SetupLock
 exit 0

package/deploy/windows/trillboards-edge.cmd

@@ -19,6 +19,25 @@ if /I "%~1"=="auto-sense" (
   if errorlevel 1 set "NEEDS_LOCAL_AI=1"
   curl.exe --fail --silent --max-time 2 http://127.0.0.1:9379/v1/models >nul 2>&1
   if errorlevel 1 set "NEEDS_LOCAL_AI=1"
+  REM A CONFIG litert-lm CANNOT PARSE IS INVISIBLE TO EVERY PROBE ABOVE. The
+  REM model is installed, the task exists, and /v1/models answers 200 -- that
+  REM route never builds an engine -- while EVERY inference returns 500. That is
+  REM exactly what shipped on 2026-08-19: setup-litertlm.ps1 wrote config.json
+  REM with `Set-Content -Encoding UTF8`, which is BOM-ful on Windows PowerShell
+  REM 5.1, and litert-lm parses the file with a plain json.load that refuses a
+  REM BOM. Without this check the repaired provisioner would land by auto-update
+  REM and never re-run, so the fix would reach NEW INSTALLS ONLY and every
+  REM already-broken box would stay broken forever while reporting healthy.
+  REM
+  REM Byte comparison, not a parse, and the BOM test comes FIRST: PowerShell's
+  REM ConvertFrom-Json happily accepts a BOM, so a parse alone would call the
+  REM broken file good. Self-clearing -- once the provisioner rewrites the file
+  REM without a BOM this never fires again -- and it fails toward REPAIR, which
+  REM is the safe direction for a box that cannot sense.
+  if exist "!LITERT_ROOT!\config.json" (
+    powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = '!LITERT_ROOT!\config.json'; $b = [System.IO.File]::ReadAllBytes($p); if ($b.Length -ge 3 -and $b[0] -eq 0xEF -and $b[1] -eq 0xBB -and $b[2] -eq 0xBF) { exit 1 }; $t = [System.Text.Encoding]::UTF8.GetString($b).Trim(); if ($t.Length -gt 0) { [void]($t | ConvertFrom-Json) }; exit 0 } catch { exit 1 }" >nul 2>&1
+    if errorlevel 1 set "NEEDS_LOCAL_AI=1"
+  )
   REM Launch the repair through the wscript host, NOT `start`. cmd's `start`
   REM allocates a NEW console for the child before PowerShell can hide itself,
   REM so the customer saw a black window on exactly the reboot that needs a