@trillboards/edge-sdk 0.2.134 → 0.2.135

package/deploy/macos/scripts/setup-litertlm.sh

@@ -125,20 +125,39 @@ CONFIG_JSON="$LITERT_DIR/config.json"
 LITERTLM_PROBE_JPEG_B64="/9j/4AAQSkZJRgABAgAAAQABAAD//gAPTGF2YzYzLjEuMTAxAP/bAEMACD4+ST5JVVVVVVVVZF1kaGhoZGRkZGhoaHBwcIODg3BwcGhocHB8fIODj5OPh4eDh5OTm5uburqystnZ4P/////EAEoAAQAAAAAAAAAAAAAAAAAAAAABAQAAAAAAAAAAAAAAAAAAAAAQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCAAgACADASIAAhEAAxEA/9oADAMBAAIRAxEAPwAAAAAAAAD/2Q=="
 
 # Write config.json. $1 = "gpu" pins the vision encoder; empty leaves it on the
-# CPU default. speculative_decoding is written EITHER WAY -- it is delegate
-# independent and byte-identical, so it survives the fallback.
+# CPU default. $2 = "true"/"false" for speculative decoding.
+#
+# BOTH ARE ALWAYS WRITTEN EXPLICITLY, AND THE SECOND ONE IS NEW.
+#
+# `speculative_decoding` used to be written unconditionally `true` under a
+# comment asserting it is "delegate independent, so it survives the fallback".
+# The evidence for that is two arms on ONE machine — an M4 Max, GPU/Metal for the
+# headline 1.433x, and its own CPU arm for the 2026-08-19 A/B in
+# `cctv-local-inference-hardware-floor.md`. Neither is a measurement of any box
+# we sell, and this file already refuses to leave an unverified `vision_backend`
+# on disk for exactly that reason. The rung below applies the same rule to this
+# key: prove it on THIS box, and back it out if the serve cannot answer with it.
+#
+# EXPLICIT `false` RATHER THAN OMITTING THE KEY, and that is measured too:
+# litert-lm 0.16.0 logs `Using speculative_decoding from config for model
+# '<id>': <bool>` on every request when the key is PRESENT and says NOTHING AT
+# ALL when it is absent (checked on a c4-standard-8, 2026-09-04). A box whose
+# state is unreadable from its own log is a box whose next incident is diagnosed
+# by guessing — which is what happened when a GCE box SEGV-looped on 2026-09-02
+# and the config had to be edited by hand to find out what it had been doing.
 write_litertlm_config() {
   _vision="$1"
+  _speculative="${2:-true}"
   mkdir -p "$LITERT_DIR" 2>/dev/null || true
   if [ -n "$_vision" ]; then
-    printf '{"models":{"%s":{"vision_backend":"%s","speculative_decoding":true}}}\n' \
-      "$MODEL_ID" "$_vision" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
+    printf '{"models":{"%s":{"vision_backend":"%s","speculative_decoding":%s}}}\n' \
+      "$MODEL_ID" "$_vision" "$_speculative" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
   else
-    printf '{"models":{"%s":{"speculative_decoding":true}}}\n' \
-      "$MODEL_ID" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
+    printf '{"models":{"%s":{"speculative_decoding":%s}}}\n' \
+      "$MODEL_ID" "$_speculative" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
   fi
   mv -f "$CONFIG_JSON.tmp" "$CONFIG_JSON" 2>/dev/null || return 1
-  log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=true)"
+  log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=$_speculative)"
 }
 
 # -- The one knob config.json cannot express -----------------------------------
@@ -237,6 +256,26 @@ verify_litertlm_vision() {
   [ "$_code" = "200" ]
 }
 
+# The same probe, but asking for NO delegate suffix rather than `,gpu`.
+#
+# WHY A SECOND FUNCTION AND NOT A FLAG: `verify_litertlm_vision` posts
+# `$MODEL_ID,gpu`, which is what the connector sends, and litert-lm returns 200
+# for a delegate token it cannot honour — so on the later rungs that suffix would
+# prove nothing about the thing being tested and could mask it. Here the model id
+# carries no suffix at all, so a 200 means the box built an engine and ran a real
+# image through it under the config now on disk, which is exactly the claim each
+# rung needs.
+verify_litertlm_plain_inference() {
+  _body="$LITERT_DIR/.plain-probe.json"
+  printf '{"model":"%s","messages":[{"role":"user","content":[{"type":"text","text":"Reply with the single word: ok"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,%s"}}]}],"max_completion_tokens":8,"temperature":0}' \
+    "$MODEL_ID" "$LITERTLM_PROBE_JPEG_B64" > "$_body" 2>/dev/null || return 1
+  _code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 180 \
+    -X POST "http://127.0.0.1:$SERVE_PORT/v1/chat/completions" \
+    -H 'Content-Type: application/json' --data-binary @"$_body" 2>/dev/null)
+  rm -f "$_body" 2>/dev/null || true
+  [ "$_code" = "200" ]
+}
+
 local_ai_healthy() {
   curl -fsS --max-time 3 "http://127.0.0.1:$SERVE_PORT/v1/models" 2>/dev/null \
     | grep -Fq "$MODEL_ID"
@@ -483,7 +522,7 @@ if [ -f "$TEMPLATE" ]; then
       -e "s#__PORT__#$SERVE_PORT#g" \
       -e "s#__LOG_DIR__#$LOG_DIR#g" \
       "$TEMPLATE" > "$TARGET" 2>/dev/null || log "WARN: could not render serve plist"
-  write_litertlm_config "gpu" || log "WARN: could not write $CONFIG_JSON - the serve will use its defaults"
+  write_litertlm_config "gpu" "true" || log "WARN: could not write $CONFIG_JSON - the serve will use its defaults"
   restart_launch_agent \
     || { [ "$REQUIRE_BUNDLED_MODEL" = "1" ] && fail "local AI service could not be started"; log "WARN: launchctl bootstrap failed for serve agent"; }
 else
@@ -491,26 +530,66 @@ else
   log "WARN: serve plist template missing at $TEMPLATE"
 fi
 
-# The GPU-vision cascade. `local_ai_healthy` only proves /v1/models answers, and
-# that route never builds an engine - so it cannot see a GPU vision encoder that
-# fails on first USE. One real image inference is the only thing that can, and if
-# it does not answer we fall back to the CPU encoder rather than leave the host
-# unable to sense at all.
+# The engine-settings cascade. `local_ai_healthy` only proves /v1/models answers,
+# and that route never builds an engine - so it cannot see a GPU vision encoder
+# that fails on first USE. One real image inference is the only thing that can,
+# and if it does not answer we drop one optimism at a time rather than leave the
+# host unable to sense at all.#
+# THREE RUNGS, NOT TWO, AND THE THIRD IS THE SAME RULE AS THE FIRST. There are
+# two optimistic engine settings in this file and only one of them was ever
+# checked. Each rung drops exactly one and re-asks the same question, so a box
+# ends on the configuration with the fewest unverified assumptions it can
+# actually serve on:
+#
+#   1. vision_backend=gpu + speculative_decoding=true   the measured fast path
+#   2.                      speculative_decoding=true   GPU vision unusable here
+#   3.                      speculative_decoding=false  neither is usable here
+#
+# Rung 3 can only turn a NON-SERVING box into a serving one: it is reached only
+# after the box has already failed rung 2, which today is the give-up path.
+# `speculative_decoding` is a draft-plus-verify scheme — strictly more compute
+# per token, needing spare parallelism to win — and every number we have for it
+# was measured on ONE machine (M4 Max: Metal for the 1.433x headline, its own CPU
+# arm for the 2026-08-19 A/B). Dropping it costs speed on a box that has already
+# failed twice, and a slow box is a box that is serving.
+_wait_for_local_ai() {
+  for _attempt in $(seq 1 60); do
+    local_ai_healthy && return 0
+    sleep 2
+  done
+  return 1
+}
+
 litertlm_vision_cascade() {
   if verify_litertlm_vision; then
     log "local AI healthy with the GPU vision encoder"
     return 0
   fi
   log "WARN: GPU vision encoder did not answer a probe inference - falling back to the CPU encoder"
-  write_litertlm_config "" || log "WARN: could not rewrite $CONFIG_JSON"
-  restart_launch_agent || log "WARN: could not restart the serve agent after the vision fallback"
-  for _attempt in $(seq 1 60); do
-    if local_ai_healthy; then
-      log "local AI healthy on the CPU vision encoder (slower, but serving)"
-      return 0
-    fi
-    sleep 2
-  done
+  write_litertlm_config "" "true" || log "WARN: could not rewrite $CONFIG_JSON"
+  # A RUNG MAY ONLY BE GRADED IF ITS CONFIG IS THE ONE RUNNING. A restart that
+  # did not happen leaves the PREVIOUS rung's engine answering, so the probe
+  # below would fail for the previous rung's reason and descending on it would
+  # disable speculative decoding without ever having tested it.
+  if ! restart_launch_agent; then
+    log "WARN: could not restart the serve agent, so the config just written is NOT the one running"
+    log "WARN: leaving the CPU vision encoder on disk unevaluated - ensure-litertlm.sh re-runs this and grades it"
+    return 1
+  fi
+  if _wait_for_local_ai && verify_litertlm_plain_inference; then
+    log "local AI healthy on the CPU vision encoder (slower, but serving)"
+    return 0
+  fi
+  log "WARN: the CPU vision encoder did not answer either - retrying without speculative decoding"
+  write_litertlm_config "" "false" || log "WARN: could not rewrite $CONFIG_JSON"
+  if ! restart_launch_agent; then
+    log "WARN: could not restart the serve agent, so the config just written is NOT the one running"
+    return 1
+  fi
+  if _wait_for_local_ai && verify_litertlm_plain_inference; then
+    log "local AI healthy WITHOUT speculative decoding (slower still, but serving)"
+    return 0
+  fi
   return 1
 }
 

package/deploy/macos/test-setup-litertlm-reinstall.sh

@@ -38,13 +38,41 @@ EOF
 
 # Fake curl models BOTH endpoints the setup script uses:
 #   GET  /v1/models            -> the catalogue body `local_ai_healthy` greps
-#   POST /v1/chat/completions  -> the vision probe, which asks for `%{http_code}`
-# TRILLBOARDS_TEST_VISION_PROBE_CODE lets a test drive the GPU-vision fallback.
+#   POST /v1/chat/completions  -> a probe inference, which asks for `%{http_code}`
+#
+# IT TELLS THE TWO PROBES APART BY THE MODEL ID IN THE BODY, and it has to: the
+# cascade's rungs differ in which id they post (`<id>,gpu` for the GPU-vision
+# rung, the bare id afterwards) and a fake that answers both identically cannot
+# exercise a cascade whose whole job is to distinguish them.
+#
+#   TRILLBOARDS_TEST_VISION_PROBE_CODE  the `,gpu` probe's status
+#   TRILLBOARDS_TEST_PLAIN_PROBE_CODE   the bare-id probe's status
+#   TRILLBOARDS_TEST_FAIL_WHEN_SPECULATIVE=1
+#       a box that cannot infer at all while `speculative_decoding` is true. The
+#       fake reads the config the script just wrote, which is the only way to
+#       exercise a rung defined by what that file says.
 cat >"$FAKE_BIN/curl" <<'EOF'
 #!/bin/bash
+body=""
+prev=""
+for arg in "$@"; do
+  case "$prev" in
+    --data-binary) body="${arg#@}" ;;
+  esac
+  prev="$arg"
+done
 for arg in "$@"; do
   if [ "$arg" = "-w" ]; then
-    printf '%s' "${TRILLBOARDS_TEST_VISION_PROBE_CODE:-200}"
+    if [ "${TRILLBOARDS_TEST_FAIL_WHEN_SPECULATIVE:-0}" = "1" ] \
+       && grep -Fq '"speculative_decoding":true' "$LITERT_LM_DIR/config.json" 2>/dev/null; then
+      printf '500'
+      exit 0
+    fi
+    if [ -n "$body" ] && grep -Fq ',gpu"' "$body" 2>/dev/null; then
+      printf '%s' "${TRILLBOARDS_TEST_VISION_PROBE_CODE:-200}"
+    else
+      printf '%s' "${TRILLBOARDS_TEST_PLAIN_PROBE_CODE:-200}"
+    fi
     exit 0
   fi
 done
@@ -147,9 +175,10 @@ grep -Fq 'falling back to the CPU encoder' <<<"$OUTPUT2"
 grep -Fq 'local AI healthy on the CPU vision encoder' <<<"$OUTPUT2"
 grep -Fq 'local AI is installed and healthy' <<<"$OUTPUT2"
 
-# The fallback must REMOVE vision_backend and KEEP speculative_decoding: the
-# latter is delegate independent and byte-identical to greedy, so it is still
-# correct on a CPU-vision host and is the only speedup that host keeps.
+# The vision fallback must REMOVE vision_backend and KEEP speculative_decoding.
+# ONE RUNG DROPS ONE OPTIMISM: this box proved it can infer with speculative
+# decoding on, so taking it away here would cost speed for a reason that was
+# just disproved on this machine.
 if grep -Fq 'vision_backend' "$LITERT_LM_DIR/config.json"; then
   printf 'FAIL: fallback left vision_backend in config.json\n' >&2
   cat "$LITERT_LM_DIR/config.json" >&2
@@ -163,5 +192,60 @@ grep -Fq '"speculative_decoding":true' "$LITERT_LM_DIR/config.json"
 
 unset TRILLBOARDS_TEST_VISION_PROBE_CODE
 
-printf 'macOS LiteRT-LM reinstall race + GPU-vision fallback regressions passed\n'
+# -- Scenario 2b: the CPU-VISION rung is verified by an INFERENCE, not by health
+#
+# `local_ai_healthy` only proves /v1/models answers, and this script's own
+# comment says that route never builds an engine. Before this, the CPU-vision
+# rung was accepted on health alone — so a box that fell back and STILL could not
+# infer logged "healthy on the CPU vision encoder (slower, but serving)" and
+# exited 0 while every frame 500s. That is how the UTF-8 BOM shipped as a green
+# install on Windows, which is the platform that already re-probes here.
+printf 'loaded\n' >"$TRILLBOARDS_TEST_STATE_DIR/service-state"
+printf '0\n' >"$TRILLBOARDS_TEST_STATE_DIR/bootstrap-attempts"
+export TRILLBOARDS_TEST_VISION_PROBE_CODE=500
+export TRILLBOARDS_TEST_PLAIN_PROBE_CODE=500
+if OUTPUT2B="$(/bin/bash "$SETUP" 2>&1)"; then
+  printf '%s\n' "$OUTPUT2B" >&2
+  printf 'FAIL: setup reported success on a box that could not infer on ANY rung\n' >&2
+  exit 1
+fi
+if grep -Fq 'local AI healthy on the CPU vision encoder' <<<"$OUTPUT2B"; then
+  printf 'FAIL: the CPU-vision rung was accepted on health alone\n' >&2
+  exit 1
+fi
+unset TRILLBOARDS_TEST_VISION_PROBE_CODE TRILLBOARDS_TEST_PLAIN_PROBE_CODE
+
+# -- Scenario 3: a host that cannot infer while speculative decoding is on -----
+#
+# `speculative_decoding` is the second optimistic engine setting this file
+# writes, and until 2026-09-04 it was written unconditionally `true` under a
+# comment calling it "delegate independent" — a property measured on ONE machine
+# (an M4 Max, both arms). This rung is the same rule the vision backend already
+# gets: prove it on THIS box, and back it out if the serve cannot answer with it.
+#
+# It can only turn a NON-SERVING box into a serving one — it is reached only
+# after the box has already failed the rung above, which used to be the give-up
+# path.
+printf 'loaded\n' >"$TRILLBOARDS_TEST_STATE_DIR/service-state"
+printf '0\n' >"$TRILLBOARDS_TEST_STATE_DIR/bootstrap-attempts"
+export TRILLBOARDS_TEST_FAIL_WHEN_SPECULATIVE=1
+
+if ! OUTPUT3="$(/bin/bash "$SETUP" 2>&1)"; then
+  printf '%s\n' "$OUTPUT3" >&2
+  printf 'FAIL: setup exited non-zero on a box that serves without speculative decoding\n' >&2
+  exit 1
+fi
+
+grep -Fq 'retrying without speculative decoding' <<<"$OUTPUT3"
+grep -Fq 'local AI healthy WITHOUT speculative decoding' <<<"$OUTPUT3"
+grep -Fq 'local AI is installed and healthy' <<<"$OUTPUT3"
+grep -Fq '"speculative_decoding":false' "$LITERT_LM_DIR/config.json"
+if grep -Fq 'vision_backend' "$LITERT_LM_DIR/config.json"; then
+  printf 'FAIL: the speculative-decoding rung left vision_backend in config.json\n' >&2
+  exit 1
+fi
+
+unset TRILLBOARDS_TEST_FAIL_WHEN_SPECULATIVE
+
+printf 'macOS LiteRT-LM reinstall race + engine-settings cascade regressions passed\n'
 

package/deploy/systemd/setup-litertlm.sh

@@ -133,20 +133,39 @@ CONFIG_JSON="$LITERT_DIR/config.json"
 LITERTLM_PROBE_JPEG_B64="/9j/4AAQSkZJRgABAgAAAQABAAD//gAPTGF2YzYzLjEuMTAxAP/bAEMACD4+ST5JVVVVVVVVZF1kaGhoZGRkZGhoaHBwcIODg3BwcGhocHB8fIODj5OPh4eDh5OTm5uburqystnZ4P/////EAEoAAQAAAAAAAAAAAAAAAAAAAAABAQAAAAAAAAAAAAAAAAAAAAAQAQAAAAAAAAAAAAAAAAAAAAARAQAAAAAAAAAAAAAAAAAAAAD/wAARCAAgACADASIAAhEAAxEA/9oADAMBAAIRAxEAPwAAAAAAAAD/2Q=="
 
 # Write config.json. $1 = "gpu" pins the vision encoder; empty leaves it on the
-# CPU default. speculative_decoding is written EITHER WAY -- it is delegate
-# independent and byte-identical, so it survives the fallback.
+# CPU default. $2 = "true"/"false" for speculative decoding.
+#
+# BOTH ARE ALWAYS WRITTEN EXPLICITLY, AND THE SECOND ONE IS NEW.
+#
+# `speculative_decoding` used to be written unconditionally `true` under a
+# comment asserting it is "delegate independent, so it survives the fallback".
+# The evidence for that is two arms on ONE machine — an M4 Max, GPU/Metal for the
+# headline 1.433x, and its own CPU arm for the 2026-08-19 A/B in
+# `cctv-local-inference-hardware-floor.md`. Neither is a measurement of any box
+# we sell, and this file already refuses to leave an unverified `vision_backend`
+# on disk for exactly that reason. The rung below applies the same rule to this
+# key: prove it on THIS box, and back it out if the serve cannot answer with it.
+#
+# EXPLICIT `false` RATHER THAN OMITTING THE KEY, and that is measured too:
+# litert-lm 0.16.0 logs `Using speculative_decoding from config for model
+# '<id>': <bool>` on every request when the key is PRESENT and says NOTHING AT
+# ALL when it is absent (checked on a c4-standard-8, 2026-09-04). A box whose
+# state is unreadable from its own log is a box whose next incident is diagnosed
+# by guessing — which is what happened when a GCE box SEGV-looped on 2026-09-02
+# and the config had to be edited by hand to find out what it had been doing.
 write_litertlm_config() {
   _vision="$1"
+  _speculative="${2:-true}"
   mkdir -p "$LITERT_DIR" 2>/dev/null || true
   if [ -n "$_vision" ]; then
-    printf '{"models":{"%s":{"vision_backend":"%s","speculative_decoding":true}}}\n' \
-      "$MODEL_ID" "$_vision" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
+    printf '{"models":{"%s":{"vision_backend":"%s","speculative_decoding":%s}}}\n' \
+      "$MODEL_ID" "$_vision" "$_speculative" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
   else
-    printf '{"models":{"%s":{"speculative_decoding":true}}}\n' \
-      "$MODEL_ID" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
+    printf '{"models":{"%s":{"speculative_decoding":%s}}}\n' \
+      "$MODEL_ID" "$_speculative" > "$CONFIG_JSON.tmp" 2>/dev/null || return 1
   fi
   mv -f "$CONFIG_JSON.tmp" "$CONFIG_JSON" 2>/dev/null || return 1
-  log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=true)"
+  log "wrote $CONFIG_JSON (vision_backend=${_vision:-cpu-default}, speculative_decoding=$_speculative)"
 }
 
 # -- The one knob config.json cannot express -----------------------------------
@@ -245,6 +264,26 @@ verify_litertlm_vision() {
   [ "$_code" = "200" ]
 }
 
+# The same probe, but asking for NO delegate suffix rather than `,gpu`.
+#
+# WHY A SECOND FUNCTION AND NOT A FLAG: `verify_litertlm_vision` posts
+# `$MODEL_ID,gpu`, which is what the connector sends, and litert-lm returns 200
+# for a delegate token it cannot honour — so on the later rungs that suffix would
+# prove nothing about the thing being tested and could mask it. Here the model id
+# carries no suffix at all, so a 200 means the box built an engine and ran a real
+# image through it under the config now on disk, which is exactly the claim each
+# rung needs.
+verify_litertlm_plain_inference() {
+  _body="$LITERT_DIR/.plain-probe.json"
+  printf '{"model":"%s","messages":[{"role":"user","content":[{"type":"text","text":"Reply with the single word: ok"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,%s"}}]}],"max_completion_tokens":8,"temperature":0}' \
+    "$MODEL_ID" "$LITERTLM_PROBE_JPEG_B64" > "$_body" 2>/dev/null || return 1
+  _code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 180 \
+    -X POST "http://127.0.0.1:$SERVE_PORT/v1/chat/completions" \
+    -H 'Content-Type: application/json' --data-binary @"$_body" 2>/dev/null)
+  rm -f "$_body" 2>/dev/null || true
+  [ "$_code" = "200" ]
+}
+
 local_ai_healthy() {
   curl -fsS --max-time 3 "http://127.0.0.1:$SERVE_PORT/v1/models" 2>/dev/null \
     | grep -Fq "$MODEL_ID"
@@ -350,15 +389,25 @@ fi
 
 # 4. Serve unit. Installed by install.sh (which has sudo); here we only start it
 # when we can, so a re-run from the service user still recovers a stopped serve.
+# RETURNS NON-ZERO WHEN IT DID NOT RESTART, and the caller has to care.
+#
+# On the ORDINARY Linux install path this function cannot restart anything:
+# `install.sh` invokes this script as the unprivileged `trillboards` service user
+# (nologin, no sudoers entry, so `sudo -n` answers "a password is required"), and
+# install.sh's own comment says so. It used to swallow that and return 0, which
+# is fine for "start it if you can" and WRONG for a cascade: every rung after the
+# first would then write a config, fail to apply it, and grade the STILL-RUNNING
+# engine — reaching a verdict about a configuration nothing ever loaded.
 restart_litertlm_serve() {
-  command -v systemctl >/dev/null 2>&1 || return 0
+  command -v systemctl >/dev/null 2>&1 || return 1
   if systemctl list-unit-files "$SERVE_UNIT.service" >/dev/null 2>&1; then
-    systemctl restart "$SERVE_UNIT" >>"$LOG" 2>&1 \
-      || sudo -n systemctl restart "$SERVE_UNIT" >>"$LOG" 2>&1 \
-      || log "WARN: could not restart $SERVE_UNIT - start it with: sudo systemctl restart $SERVE_UNIT"
-  else
-    log "WARN: $SERVE_UNIT.service is not installed yet (install.sh installs it)"
+    systemctl restart "$SERVE_UNIT" >>"$LOG" 2>&1 && return 0
+    sudo -n systemctl restart "$SERVE_UNIT" >>"$LOG" 2>&1 && return 0
+    log "WARN: could not restart $SERVE_UNIT - start it with: sudo systemctl restart $SERVE_UNIT"
+    return 1
   fi
+  log "WARN: $SERVE_UNIT.service is not installed yet (install.sh installs it)"
+  return 1
 }
 
 wait_for_litertlm_health() {
@@ -373,7 +422,7 @@ wait_for_litertlm_health() {
 # Ask for GPU vision BEFORE the restart so the engine builds with it. The engine
 # is constructed lazily on the first inference, so config.json must be on disk
 # before that request, not merely before the next boot.
-write_litertlm_config "gpu" || log "WARN: could not write $CONFIG_JSON - the serve will use its defaults"
+write_litertlm_config "gpu" "true" || log "WARN: could not write $CONFIG_JSON - the serve will use its defaults"
 
 restart_litertlm_serve
 
@@ -384,16 +433,52 @@ if [ "$HEALTHY" -eq 1 ]; then
   # The cascade. `local_ai_healthy` only proves /v1/models answers, which it does
   # without ever building an engine - so it CANNOT see a GPU vision encoder that
   # fails on first use. One real image inference is the only thing that can.
+#
+# THREE RUNGS, NOT TWO, AND THE THIRD IS THE SAME RULE AS THE FIRST. There are
+# two optimistic engine settings in this file and only one of them was ever
+# checked. Each rung drops exactly one and re-asks the same question, so a box
+# ends on the configuration with the fewest unverified assumptions it can
+# actually serve on:
+#
+#   1. vision_backend=gpu + speculative_decoding=true   the measured fast path
+#   2.                      speculative_decoding=true   GPU vision unusable here
+#   3.                      speculative_decoding=false  neither is usable here
+#
+# Rung 3 can only turn a NON-SERVING box into a serving one: it is reached only
+# after the box has already failed rung 2, which today is the give-up path.
+# `speculative_decoding` is a draft-plus-verify scheme — strictly more compute
+# per token, needing spare parallelism to win — and every number we have for it
+# was measured on ONE machine (M4 Max: Metal for the 1.433x headline, its own CPU
+# arm for the 2026-08-19 A/B). Dropping it costs speed on a box that has already
+# failed twice, and a slow box is a box that is serving.
   if verify_litertlm_vision; then
     log "local AI healthy with the GPU vision encoder - the connector will run ON-DEVICE"
   else
     log "WARN: GPU vision encoder did not answer a probe inference - falling back to the CPU encoder"
-    write_litertlm_config "" || log "WARN: could not rewrite $CONFIG_JSON"
-    restart_litertlm_serve
-    if wait_for_litertlm_health; then
+    write_litertlm_config "" "true" || log "WARN: could not rewrite $CONFIG_JSON"
+    # A RUNG MAY ONLY BE GRADED IF ITS CONFIG IS THE ONE RUNNING. On the ordinary
+    # Linux install path this script runs unprivileged and cannot restart the
+    # unit, so the probe below would interrogate the engine built for the
+    # PREVIOUS rung and fail for the previous rung's reason - descending on that
+    # would disable speculative decoding without ever having tested it. So when
+    # the restart did not happen we STOP here, leaving the CPU-vision config on
+    # disk exactly as before: `install.sh` applies it with its own privileged
+    # restart and runs this script again, and that pass can restart and grade.
+    if ! restart_litertlm_serve; then
+      log "WARN: the serve could not be restarted here, so the config just written is NOT the one running"
+      log "WARN: leaving the CPU vision encoder on disk unevaluated - install.sh applies it and the next pass grades it"
+    elif wait_for_litertlm_health && verify_litertlm_plain_inference; then
       log "local AI healthy on the CPU vision encoder (slower, but serving)"
     else
-      HEALTHY=0
+      log "WARN: the CPU vision encoder did not answer either - retrying without speculative decoding"
+      write_litertlm_config "" "false" || log "WARN: could not rewrite $CONFIG_JSON"
+      if ! restart_litertlm_serve; then
+        log "WARN: the serve could not be restarted here, so the config just written is NOT the one running"
+      elif wait_for_litertlm_health && verify_litertlm_plain_inference; then
+        log "local AI healthy WITHOUT speculative decoding (slower still, but serving)"
+      else
+        HEALTHY=0
+      fi
     fi
   fi
 fi
@@ -412,9 +497,14 @@ if [ "$HEALTHY" -ne 1 ]; then
   # works, which is the right trade for a value we could not check: a box that is
   # slow is a box that is serving, and `setup-litertlm.sh` re-runs on every
   # install.sh, which will promote it to GPU the moment it can prove GPU works.
-  write_litertlm_config "" \
+  write_litertlm_config "" "false" \
     || log "WARN: could not rewrite $CONFIG_JSON to the CPU default"
-  log "vision_backend left UNVERIFIED, so it was reset to the CPU encoder"
+  # BOTH optimisms are reset, for one reason: neither was checked. The serve
+  # never came up, so the cascade that would have proved either of them never
+  # ran, and the file OUTLIVES this install — the unit is enabled, so the next
+  # boot starts on a configuration nobody confirmed. `setup-litertlm.sh` re-runs
+  # on every install.sh and promotes both again the moment it can prove them.
+  log "vision_backend and speculative_decoding left UNVERIFIED, so both were reset to the conservative default"
 
   # NOT fatal. auto-sense waits and beacons, so an install that gets this far
   # with the model on disk recovers by itself once the serve comes up.

package/deploy/windows/setup-litertlm.ps1

@@ -166,17 +166,33 @@ $script:LitertlmProbeJpegB64 = "/9j/4AAQSkZJRgABAgAAAQABAAD//gAPTGF2YzYzLjEuMTAx
 function Get-LitertlmConfigPath { Join-Path $LitertDir "config.json" }
 
 # $VisionBackend = "gpu" pins the vision encoder; "" leaves it on the CPU
-# default. speculative_decoding is written EITHER WAY -- it is delegate
-# independent, so it survives the fallback.
+# default. $SpeculativeDecoding is the second engine setting this file writes.
+#
+# BOTH ARE ALWAYS WRITTEN EXPLICITLY, AND THE SECOND ONE IS NEW.
+#
+# speculative_decoding used to be written unconditionally $true under a comment
+# asserting it is "delegate independent, so it survives the fallback". The
+# evidence for that is two arms on ONE machine - an M4 Max, Metal for the
+# headline 1.433x and its own CPU arm for the 2026-08-19 A/B in
+# cctv-local-inference-hardware-floor.md. Neither is a measurement of any box we
+# sell, and this file already refuses to leave an unverified vision_backend on
+# disk for exactly that reason. The cascade below applies the same rule here.
+#
+# EXPLICIT $false RATHER THAN OMITTING THE KEY, and that is measured too:
+# litert-lm 0.16.0 logs "Using speculative_decoding from config for model
+# '<id>': <bool>" on every request when the key is PRESENT and says NOTHING AT
+# ALL when it is absent (checked on a c4-standard-8, 2026-09-04). A box whose
+# state is unreadable from its own log is a box whose next incident is diagnosed
+# by guessing.
 function Write-LitertlmConfig {
-  param([string]$VisionBackend = "")
+  param([string]$VisionBackend = "", [bool]$SpeculativeDecoding = $true)
   try {
     if (-not (Test-Path -LiteralPath $LitertDir)) {
       New-Item -ItemType Directory -Force -Path $LitertDir | Out-Null
     }
     $model = [ordered]@{}
     if ($VisionBackend) { $model["vision_backend"] = $VisionBackend }
-    $model["speculative_decoding"] = $true
+    $model["speculative_decoding"] = $SpeculativeDecoding
     $cfg = [ordered]@{ models = [ordered]@{ $ModelId = $model } }
     $path = Get-LitertlmConfigPath
     $tmp = "$path.tmp"
@@ -198,7 +214,7 @@ function Write-LitertlmConfig {
       (New-Object System.Text.UTF8Encoding($false)))
     Move-Item -LiteralPath $tmp -Destination $path -Force
     $label = if ($VisionBackend) { $VisionBackend } else { "cpu-default" }
-    Log "wrote $path (vision_backend=$label, speculative_decoding=true)"
+    Log "wrote $path (vision_backend=$label, speculative_decoding=$SpeculativeDecoding)"
     return $true
   } catch {
     Log "WARN: could not write litert-lm config.json: $_"
@@ -670,7 +686,7 @@ if (-not (Test-Path -LiteralPath $registration)) {
 
 # Ask for GPU vision BEFORE the serve starts: the engine is built lazily on the
 # first inference, so config.json must be on disk before that request.
-Write-LitertlmConfig -VisionBackend "gpu" | Out-Null
+Write-LitertlmConfig -VisionBackend "gpu" -SpeculativeDecoding $true | Out-Null
 
 Log "registering litert-lm serve Scheduled Task on :$Port"
 & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $registration -LitertLmPath $litertlmPath -Port $Port *>> $LogPath
@@ -698,7 +714,7 @@ if (-not $healthy) {
 # encoder rather than leave this host unable to sense at all.
 if (-not (Test-LitertlmVision -InferenceModel "$ModelId,gpu")) {
   Log "WARN: GPU vision encoder did not answer a probe inference; falling back to the CPU encoder"
-  Write-LitertlmConfig | Out-Null
+  Write-LitertlmConfig -SpeculativeDecoding $true | Out-Null
   & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $registration -LitertLmPath $litertlmPath -Port $Port *>> $LogPath
   $healthy = $false
   for ($attempt = 0; $attempt -lt 60 -and -not $healthy; $attempt++) {
@@ -730,9 +746,46 @@ if (-not (Test-LitertlmVision -InferenceModel "$ModelId,gpu")) {
   } elseif (Test-LitertlmVision -InferenceModel $ModelId) {
     Log "local AI healthy on the full CPU fallback (slower, but serving)"
   } else {
-    Fail-LocalAi "Gemma is installed and the local AI service is answering, but it could not analyze a test image with either GPU inference or the full CPU fallback. See $LogPath."
-    Stop-SetupLock
-    exit 1
+    # THE THIRD RUNG, AND IT IS THE SAME RULE AS THE FIRST. There are two
+    # optimistic engine settings in this file and only one of them was ever
+    # checked. Each rung drops exactly one and re-asks the same question, so a
+    # box ends on the configuration with the fewest unverified assumptions it can
+    # actually serve on:
+    #
+    #   1. vision_backend=gpu + speculative_decoding=true   the measured fast path
+    #   2.                      speculative_decoding=true   GPU vision unusable here
+    #   3.                      speculative_decoding=false  neither is usable here
+    #
+    # This can only turn a box that was about to FAIL THE INSTALL into a serving
+    # one: it is reached only where the previous code called Fail-LocalAi.
+    # speculative_decoding is a draft-plus-verify scheme - strictly more compute
+    # per token, needing spare parallelism to win - and every number we have for
+    # it was measured on ONE machine. Dropping it costs speed on a box that has
+    # already failed twice, and a slow box is a box that is serving.
+    Log "WARN: neither GPU inference nor the full CPU fallback answered; retrying without speculative decoding"
+    Write-LitertlmConfig -SpeculativeDecoding $false | Out-Null
+    # A RUNG MAY ONLY BE GRADED IF ITS CONFIG IS THE ONE RUNNING. Re-registering
+    # the Scheduled Task is what applies the file; if that fails, the engine
+    # still answering is the previous rung's and grading it would reach a verdict
+    # about a configuration nothing ever loaded.
+    & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $registration -LitertLmPath $litertlmPath -Port $Port *>> $LogPath
+    if ($LASTEXITCODE -ne 0) {
+      Fail-LocalAi "Gemma is installed, but the local AI service could not be restarted to apply the fallback configuration. Restart Windows, then retry Local AI. See $LogPath."
+      Stop-SetupLock
+      exit 1
+    }
+    $healthy = $false
+    for ($attempt = 0; $attempt -lt 60 -and -not $healthy; $attempt++) {
+      $healthy = Test-ServeHealthy
+      if (-not $healthy) { Start-Sleep -Seconds 2 }
+    }
+    if ($healthy -and (Test-LitertlmVision -InferenceModel $ModelId)) {
+      Log "local AI healthy WITHOUT speculative decoding (slower still, but serving)"
+    } else {
+      Fail-LocalAi "Gemma is installed and the local AI service is answering, but it could not analyze a test image with GPU inference, the full CPU fallback, or with speculative decoding disabled. See $LogPath."
+      Stop-SetupLock
+      exit 1
+    }
   }
 } else {
   Log "local AI healthy with the GPU vision encoder"

package/dist/EdgeAgent.js

@@ -44,7 +44,7 @@ const edge_ads_1 = require("@trillboards/edge-ads");
 const CommandRouter_1 = require("./CommandRouter");
 const config_1 = require("./config");
 const dataDir_1 = require("./dataDir");
-const SDK_VERSION = '0.2.134';
+const SDK_VERSION = '0.2.135';
 class EdgeAgent extends events_1.EventEmitter {
     constructor(config, deps) {
         super();

package/dist/diagnose.d.ts

@@ -31,7 +31,7 @@ export interface DiagnosticReport {
         skipped: number;
     };
 }
-declare const SDK_VERSION = "0.2.134";
+declare const SDK_VERSION = "0.2.135";
 declare function checkNodeVersion(): CheckResult;
 declare function checkPlatform(): CheckResult;
 declare function checkMemory(): CheckResult;

package/dist/diagnose.js

@@ -63,7 +63,7 @@ const localNetworkAccess_1 = require("./localNetworkAccess");
 // ---------------------------------------------------------------------------
 // SDK version — mirrors package.json; kept in sync by CI
 // ---------------------------------------------------------------------------
-const SDK_VERSION = '0.2.134';
+const SDK_VERSION = '0.2.135';
 exports._SDK_VERSION = SDK_VERSION;
 // ---------------------------------------------------------------------------
 // ANSI helpers (no external deps)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trillboards/edge-sdk",
-  "version": "0.2.134",
+  "version": "0.2.135",
   "description": "Composable cross-platform Edge AI SDK for audience sensing on DOOH/CTV devices",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",