failproofai 0.0.14-beta.2 → 0.0.15-beta.0

package/.next/standalone/integration-suite/README.md

@@ -45,6 +45,48 @@ failproofai's HEAD changed since its last green run — unchanged ⇒ can't have
 drifted ⇒ skip, protecting LLM/vendor quota. `FAIL`/`INCONCLUSIVE`/`ERROR` always
 re-probe until they recover. Dispatch with **force** to probe everything.
 
+## Two channels: `stable` and `beta`
+
+The workflow runs as a **matrix over `CANARY_CHANNEL`**, in two concurrent legs
+answering different questions:
+
+| Leg | Installs | Question | On failure |
+|-----|----------|----------|------------|
+| `stable` | what users get, all 12 CLIs | *is enforcement broken now?* | **fails the job** + Slack |
+| `beta` | each vendor's public pre-release ref, 6 CLIs | *is it about to break?* | **advisory only** — never fails |
+
+Only six vendors publish something usable ahead of their release, so the beta leg
+skips the rest rather than re-installing a stable build and reporting it as
+pre-release coverage:
+
+| CLI | Pre-release ref | Typical lead |
+|-----|-----------------|--------------|
+| codex | npm `alpha` | up to ~12 d (minor bumps only — patches ship blind) |
+| copilot | npm `prerelease` | 0.6–5.8 d |
+| openclaw | npm `beta` | 0.3–11.4 d |
+| cursor | `install?channel=lab` | 2–4 d |
+| goose | `CANARY=true` (rolling tag) | ~1 release cycle |
+| claude | `latest` — **inverted**, see below | ~13 d |
+
+**claude runs backwards.** Nothing ships ahead of `latest`, which *is* the
+bleeding edge (~1 release/day); what exists is `stable`, ~13 days behind. So the
+stable leg pins `bash -s stable` — testing what conservative users run, and
+stopping a same-day Anthropic release from red-lighting an unrelated PR — and
+`latest` becomes the early-warning ref. **hermes** is inverted too but has no fix:
+its installer git-clones `main`, which is also what every hermes user gets, so
+there is nothing ahead of us to probe.
+
+**Escalation is a cross-leg comparison, not a beta failure.** A CLI red on both
+legs is already broken and belongs to the stable leg's alarm. The early warning is
+`stable green + beta not-green`, held for **two consecutive runs** so a broken
+alpha that gets reverted before release doesn't burn attention. Note the signal is
+usually `INCONCLUSIVE`/`ERROR`, not `FAIL` — a vendor payload change stops the
+model before it ever calls a tool — so a FAIL-only rule would miss it entirely.
+
+Each leg has its own Docker volume and Actions cache key: a pre-release install
+overwrites the stable binary in a shared `$HOME`, and a beta result must never be
+able to overwrite the stable leg's gating record.
+
 ## Auth & secrets (the `cli-integration` Environment)
 
 > The GitHub **Environment** is still named `cli-integration` — it's configured in
@@ -77,7 +119,7 @@ re-login on the capture machine and refresh the secret.
 ```
 ci-entrypoint.sh      CI front door: build -> sandbox -> install -> probe -> cleanup
 Dockerfile            non-root sandbox base image
-install-clis.sh       install/upgrade all 12 CLIs @latest (with retries)
+install-clis.sh       install/upgrade the CLIs for $CANARY_CHANNEL (with retries)
 inject-tokens.sh      write captured OAuth creds into the fresh volume
 probe-cli.sh          live enforcement probe for ONE CLI (the oracle)
 canary-policies.mjs   benign-marker custom policies the probe trips

package/.next/standalone/integration-suite/ci-entrypoint.sh

@@ -27,6 +27,7 @@
 #   CANARY_LLM_MODEL       (default deepseek-v4-pro)
 #   CANARY_CLAUDE_MODEL    (default claude-haiku-4-5)
 #   CANARY_PI_MODEL        (default claude-haiku-4-5)
+#   CANARY_CODEX_MODEL     (default gpt-5.1-codex-mini)
 #   COPILOT_GITHUB_TOKEN   copilot PAT
 #   CURSOR_TOKEN_TGZ_B64   } base64 gzip-tars of each OAuth credential tree,
 #   DEVIN_TOKEN_TGZ_B64    } rooted at $HOME (see capture-tokens.sh)
@@ -43,11 +44,20 @@ set -u
 HERE="$(cd "$(dirname "$0")" && pwd)"
 REPO="${GITHUB_WORKSPACE:-$(dirname "$HERE")}"
 
-VOL="${CANARY_VOL:-integration-suite}"
+# Each channel gets its OWN volume: installing a pre-release into the same $HOME
+# overwrites the stable binary (npm -g, and the vendor installers all rewrite
+# their own shims), so the two refs cannot coexist in one volume.
+CHANNEL="${CANARY_CHANNEL:-stable}"
+SUFFIX=""; [ "$CHANNEL" != stable ] && SUFFIX="-$CHANNEL"
+VOL="${CANARY_VOL:-integration-suite$SUFFIX}"
 IMAGE="${CANARY_IMAGE:-failproofai-integration-suite:base}"
-ENVFILE="${CANARY_ENVFILE:-$REPO/canary.env}"
-TOKENS_DIR="${CANARY_TOKENS_DIR:-$REPO/tokens}"
-STATE="${CANARY_STATE:-$REPO/integration-suite-state.json}"
+ENVFILE="${CANARY_ENVFILE:-$REPO/canary$SUFFIX.env}"
+TOKENS_DIR="${CANARY_TOKENS_DIR:-$REPO/tokens$SUFFIX}"
+STATE="${CANARY_STATE:-$REPO/integration-suite-state$SUFFIX.json}"
+# The stable leg's state, so the beta leg can tell "about to break" from
+# "already broken" (see report.js). Unset on the stable leg.
+PEER_STATE="${CANARY_PEER_STATE:-}"
+[ "$CHANNEL" != stable ] && [ -z "$PEER_STATE" ] && PEER_STATE="$REPO/integration-suite-state.json"
 
 step() { echo "── $* ──" >&2; }
 
@@ -123,8 +133,8 @@ docker volume rm "$VOL" -f >/dev/null 2>&1 || true
 docker volume create "$VOL" >/dev/null || { echo "✗ volume create failed" >&2; exit 1; }
 
 # ── 4. install CLIs @latest, then inject tokens ─────────────────────────────
-step "installing 12 CLIs @latest into the fresh volume"
-docker run --rm -v "$VOL:/home/canary" -v "$HERE:/opt/canary:ro" \
+step "installing CLIs (channel=$CHANNEL) into the fresh volume"
+docker run --rm -e CANARY_CHANNEL="$CHANNEL" -v "$VOL:/home/canary" -v "$HERE:/opt/canary:ro" \
   "$IMAGE" bash /opt/canary/install-clis.sh \
   || { echo "✗ CLI install failed" >&2; exit 1; }
 
@@ -145,6 +155,9 @@ step "assembling gateway env-file"
     echo "CANARY_LLM_MODEL=${CANARY_LLM_MODEL:-deepseek-v4-pro}"
     echo "CANARY_CLAUDE_MODEL=${CANARY_CLAUDE_MODEL:-claude-haiku-4-5}"
     echo "CANARY_PI_MODEL=${CANARY_PI_MODEL:-claude-haiku-4-5}"
+    # Without this line probe-cli.sh's CANARY_CODEX_MODEL override was dead plumbing:
+    # the var is read inside the container, and only what is written here crosses into it.
+    echo "CANARY_CODEX_MODEL=${CANARY_CODEX_MODEL:-gpt-5.1-codex-mini}"
     echo "COPILOT_GITHUB_TOKEN=${COPILOT_GITHUB_TOKEN:-}"
   } > "$ENVFILE"
 )
@@ -161,4 +174,6 @@ CANARY_VOL="$VOL" \
 CANARY_IMAGE="$IMAGE" \
 CANARY_STATE="$STATE" \
 CANARY_ENVFILE="$ENVFILE" \
+CANARY_CHANNEL="$CHANNEL" \
+CANARY_PEER_STATE="$PEER_STATE" \
   bash "$HERE/run.sh" ${CANARY_CLIS:-}

package/.next/standalone/integration-suite/install-clis.sh

@@ -2,10 +2,22 @@
 # ─────────────────────────────────────────────────────────────────────────────
 # Tier-0 install probe — runs INSIDE the canary sandbox container.
 #
-# Installs/upgrades all 12 agent CLIs to @latest into the persistent HOME volume,
-# then records which resolve to a runnable binary + their version. This is both
-# the daily "fresh upgrade" step AND the Tier-0 canary signal (packaging/binary
-# breaks show up here as a FAIL).
+# Installs/upgrades the agent CLIs into the persistent HOME volume, then records
+# which resolve to a runnable binary + their version. This is both the daily
+# "fresh upgrade" step AND the Tier-0 canary signal (packaging/binary breaks show
+# up here as a FAIL).
+#
+# CANARY_CHANNEL selects WHICH ref of each CLI to install:
+#   stable (default) — what users get. All 12 CLIs.
+#   beta             — the vendor's public pre-release ref, for early warning.
+#                      Only the CLIs that actually publish one; the rest are
+#                      SKIPPED, never silently re-installed at stable (that would
+#                      be duplicate work reported as coverage).
+#
+# Every beta ref below was verified live against the vendor's registry/CDN on
+# 2026-07-23; see ~/Desktop/failproofai-integration-suite-beta-channels-design.md
+# for the evidence, lead times, and the CLIs that have no pre-release ref at all
+# (factory, antigravity, pi, opencode, devin, hermes).
 #
 # Install methods verified 2026-07-16 (see failproofai-cli-canary-design.md).
 # Vendor installers scatter binaries across several dirs, so we search an
@@ -18,6 +30,7 @@ set -u
 
 RESULTS="${CANARY_RESULTS:-$HOME/canary-tier0.json}"
 LOGDIR="${CANARY_LOGDIR:-$HOME/canary-logs}"
+CHANNEL="${CANARY_CHANNEL:-stable}"
 mkdir -p "$LOGDIR"
 
 # Cover every dir the vendor installers are known to drop binaries into.
@@ -25,8 +38,19 @@ export PATH="$HOME/.npm-global/bin:$HOME/.local/bin:$HOME/.factory/bin:$HOME/.he
 
 ROWS=()
 
+# probe <id> <binary> <--version flag> <stable install cmd> [beta install cmd]
+# Omitting the 5th argument declares "this CLI has no public pre-release ref",
+# which skips it entirely on the beta channel.
 probe() {
-  local id="$1" bin="$2" vflag="$3" install="$4"
+  local id="$1" bin="$2" vflag="$3" install="$4" beta="${5:-}"
+  if [ "$CHANNEL" != stable ]; then
+    if [ -z "$beta" ]; then
+      echo "════════════════════ $id ════════════════════"
+      echo "  skipped — no public pre-release ref for this CLI"
+      return 0
+    fi
+    install="$beta"
+  fi
   echo "════════════════════ $id ════════════════════"
   local t0 t1 rc=1; t0=$SECONDS
   # Retry transient install failures. Flaky vendor CDNs / HTTP-2 stream resets are
@@ -64,19 +88,40 @@ probe() {
     "$id" "$status" "${resolved:-}" "${ver:-}" "$rc" "$((t1 - t0))")")
 }
 
-#      id           binary         --ver flag    install command (@latest)
-probe  claude       claude         "--version"   'curl -fsSL https://claude.ai/install.sh | bash'
-probe  codex        codex          "--version"   'npm install -g @openai/codex@latest'
-probe  copilot      copilot        "--version"   'npm install -g @github/copilot@latest'
-probe  cursor       cursor-agent   "--version"   'curl https://cursor.com/install -fsS | bash'
+#      id           binary         --ver flag    stable ref                                          beta ref (omitted = none exists)
+#
+# claude is the one CLI whose channels run BACKWARDS: nothing ships ahead of
+# `latest`, which is the bleeding edge (~1 release/day). What exists is `stable`,
+# ~13 days behind. So the stable leg pins `stable` — both to test what
+# conservative users actually run and to stop a same-day Anthropic release
+# red-lighting an unrelated PR — and `latest` becomes the early-warning ref.
+probe  claude       claude         "--version"   'curl -fsSL https://claude.ai/install.sh | bash -s stable' \
+                                                 'curl -fsSL https://claude.ai/install.sh | bash'
+probe  codex        codex          "--version"   'npm install -g @openai/codex@latest' \
+                                                 'npm install -g @openai/codex@alpha'
+probe  copilot      copilot        "--version"   'npm install -g @github/copilot@latest' \
+                                                 'npm install -g @github/copilot@prerelease'
+# Cursor validates the channel server-side: only prod/lab/static/prod-stable-internal
+# return 200, anything else is a 400. `lab` is Anysphere's own dogfood channel.
+probe  cursor       cursor-agent   "--version"   'curl https://cursor.com/install -fsS | bash' \
+                                                 'curl "https://cursor.com/install?channel=lab" -fsS | bash'
+# opencode's beta/dev tags are ~31 branch snapshots a day (0.0.0-<branch>-<ts>),
+# not release candidates — deliberately no beta ref.
 probe  opencode     opencode       "--version"   'npm install -g opencode-ai@latest'
 probe  pi           pi             "--version"   'npm install -g @mariozechner/pi-coding-agent@latest'
+# hermes install.sh git-clones main, which is ALSO what every hermes user gets —
+# its tags are markers nobody installs. So there is nothing ahead of us to probe,
+# and pinning a tag would test something no user runs.
 probe  hermes       hermes         "--version"   'curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash'
-probe  openclaw     openclaw       "--version"   'npm install -g openclaw@latest'
+probe  openclaw     openclaw       "--version"   'npm install -g openclaw@latest' \
+                                                 'npm install -g openclaw@beta'
 probe  factory      droid          "--version"   'curl -fsSL https://app.factory.ai/cli | sh'
 probe  devin        devin          "--version"   'curl -fsSL https://cli.devin.ai/install.sh | bash'
 probe  antigravity  agy            "--version"   'curl -fsSL https://antigravity.google/cli/install.sh | bash'
-probe  goose        goose          "--version"   'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash'
+# goose's canary is a ROLLING TAG (prerelease:false), selected by env var — not a
+# GitHub prerelease, so channel detection that scans for prerelease:true misses it.
+probe  goose        goose          "--version"   'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash' \
+                                                 'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CANARY=true CONFIGURE=false bash'
 
 # Emit JSON array.
 { printf '['; IFS=,; printf '%s' "${ROWS[*]}"; printf ']\n'; } > "$RESULTS"
@@ -87,5 +132,5 @@ ok=0; fail=0
 for r in "${ROWS[@]}"; do
   case "$r" in *'"status":"OK"'*) ok=$((ok+1));; *) fail=$((fail+1));; esac
 done
-printf 'installed OK: %s / 12   failed: %s\n' "$ok" "$fail"
+printf 'channel: %s   installed OK: %s / %s   failed: %s\n' "$CHANNEL" "$ok" "$((ok + fail))" "$fail"
 echo "results JSON: $RESULTS"

package/.next/standalone/integration-suite/probe-cli.sh

@@ -14,14 +14,33 @@
 set -u
 CLI="${1:?usage: probe-cli.sh <cli>}"
 : "${CANARY_LLM_API_KEY:?gateway key missing}"
-# Gateway default model. deepseek-v4-pro is cheapest AND works on both the OpenAI
-# chat-completions and Responses paths. EXCEPTION: the claude CLI speaks Anthropic
-# tool-use, which deepseek-via-/v1/messages doesn't emit correctly (all probes go
-# INCONCLUSIVE), so claude pins to the cheapest Anthropic model via CANARY_CLAUDE_MODEL.
+# Gateway default model, used by every CLI without a pin of its own. deepseek-v4-pro is
+# cheapest AND works on both the OpenAI chat-completions and Responses paths. Three CLIs
+# are pinned away from it below (claude, pi, codex) — each for a payload deepseek refuses.
+# EXCEPTION 1: the claude CLI speaks Anthropic tool-use, which deepseek-via-/v1/messages
+# doesn't emit correctly (all probes go INCONCLUSIVE), so claude pins to the cheapest
+# Anthropic model via CANARY_CLAUDE_MODEL.
 : "${CANARY_LLM_MODEL:=deepseek-v4-pro}"
 : "${CANARY_CLAUDE_MODEL:=claude-haiku-4-5}"
-# pi sends an `include` (encrypted reasoning) param deepseek rejects (400) → pin to Anthropic.
+# EXCEPTION 2: pi sends an `include` (encrypted reasoning) param deepseek rejects (400).
 : "${CANARY_PI_MODEL:=claude-haiku-4-5}"
+# EXCEPTION 3: codex ≥0.145.0 hits the SAME rejection. For a model it has no metadata for
+# (deepseek logs "Model metadata not found. Defaulting to fallback metadata") it now sends
+# `reasoning:{summary:"auto"}` + `include:["reasoning.encrypted_content"]`, where 0.144.6
+# sent `reasoning:null` + `include:[]`. The gateway answers 400 "Encrypted content is not
+# supported with this model" (param: include), codex exits before its first tool call, and
+# BOTH probes report INCONCLUSIVE — enforcement is fine, there is just nothing to observe.
+# No config override strips it (`model_reasoning_summary=none`,
+# `model_supports_reasoning_summaries=false`, `model_reasoning_effort=none` all still emit
+# `include`), and the old escape hatch is gone — `wire_api="chat"` is rejected outright in
+# 0.145.0 (openai/codex#7782). So codex needs a model that accepts encrypted reasoning
+# content. gpt-5.1-codex-mini is the cheapest that does AND supports codex's full toolset:
+# gpt-5.4-nano accepts the reasoning params but 400s on `tool_search`.
+# Do NOT "fix" this by pinning codex to a Claude model the way pi and claude are: the
+# gateway routes Anthropic weighted 1:1 through Bedrock, which 400s on codex's request
+# metadata (#576). That fails on roughly half of requests — a coin-flip red is worse in a
+# daily canary than a consistent one, and a single green probe does not disprove it.
+: "${CANARY_CODEX_MODEL:=gpt-5.1-codex-mini}"
 # Gateway base URL — overridable via env (CI supplies it as a secret). Strip any
 # trailing slash so the `$GW/v1` joins below never produce `//v1`.
 GW="${CANARY_LLM_BASE_URL:-https://models.aikin.club}"; GW="${GW%/}"
@@ -129,7 +148,7 @@ drive() { # $1 = prompt ; run ONE prompt headless, executing tools without appro
     codex)    ( cd "$BASE" && codex exec --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust \
                   -c model_providers.gw.name="gw" -c model_providers.gw.base_url="$GW/v1" -c model_providers.gw.wire_api="responses" \
                   -c model_providers.gw.env_key="CANARY_LLM_API_KEY" -c model_provider="gw" \
-                  -c model="${CANARY_CODEX_MODEL:-deepseek-v4-pro}" "$1" 2>&1 ) ;;  # Claude models 400 on Bedrock metadata; deepseek is non-Bedrock
+                  -c model="$CANARY_CODEX_MODEL" "$1" 2>&1 ) ;;
     cursor)   ( cd "$BASE" && cursor-agent -p --force "$1" 2>&1 ) ;;
     copilot)  ( cd "$BASE" && copilot -p "$1" --allow-all-tools 2>&1 ) ;;
     devin)    ( cd "$BASE" && devin -p "$1" --permission-mode dangerous --respect-workspace-trust false 2>&1 ) ;;
@@ -163,7 +182,21 @@ read_denied() { grep -qE "result=deny policy=(failproofai/|custom/)?(canary-read
 # Vendor quota / auth errors (Copilot-Free credits, antigravity Google quota,
 # expired logins) → the CLI errors before any tool call. Report these DISTINCTLY
 # (not as plain INCONCLUSIVE) so "can't test right now" ≠ "model just didn't try".
-is_error() { printf '%s' "$1" | grep -qiE "quota|rate.?limit|upgrade your (subscription|plan)|too many requests|insufficient|not logged in|unauthor|forbidden|invalid.*(key|token|credential)|payment required|\\b(401|402|429)\\b"; }
+# Payload rejections belong in the same bucket: when a CLI update starts sending a param
+# the pinned model refuses, the run is untestable, NOT a model that declined to act.
+# Leaving those as INCONCLUSIVE is how codex 0.145.0's `include:
+# ["reasoning.encrypted_content"]` 400 read as a quiet 🟡 for a full day instead of the
+# ⚠️ it was.
+#
+# These patterns MUST stay machine-shaped. `$1` is the agent's whole transcript, so a bare
+# `400` or `not supported` also matches ordinary prose ("400 tests passed", "that flag is
+# not supported") and would report a chatty refusal as a vendor outage — the exact
+# inversion this function exists to prevent. Hence: the structured `"code": 400` form
+# rather than a loose `400`, and `not supported with` (the gateway's own phrasing:
+# "not supported with this model") rather than `not supported`. Both live failures we have
+# seen carry `invalid_request_error` AND `BadRequestError` anyway, so the tight forms lose
+# nothing. __tests__/integration-suite/is-error.test.ts holds the fixtures both ways.
+is_error() { printf '%s' "$1" | grep -qiE "quota|rate.?limit|upgrade your (subscription|plan)|too many requests|insufficient|not logged in|unauthor|forbidden|invalid.*(key|token|credential)|payment required|\\b(401|402|429)\\b|\"code\"[[:space:]]*:[[:space:]]*\"?40[0-9]|bad.?request(error)?\\b|invalid_request_error|not supported with|unsupported (parameter|model|value)|deploymentnotfound"; }
 
 ATTEMPTS=3   # retry up to N times to absorb LLM nondeterminism (flaky tool-callers)
 
@@ -180,8 +213,13 @@ for _ in $(seq 1 $ATTEMPTS); do
   denied canary-bash "$LOGA/hooks.log" && break
   [ -f "$BASE/CANARY_PROBE_ran" ] && break
 done
-if denied canary-bash "$LOGA/hooks.log"; then VA=PASS
-elif [ -f "$BASE/CANARY_PROBE_ran" ]; then VA=FAIL
+# Ground truth FIRST. A CLI that logs our deny and runs the command anyway is
+# the silent-allow this suite exists to catch (copilot 1.0.70 did exactly that)
+# — and checking our own hooks.log first scored it PASS, because our log says
+# "denied" whether or not the CLI honoured it. The marker file is the only
+# evidence of what the CLI actually did, so it decides.
+if [ -f "$BASE/CANARY_PROBE_ran" ]; then VA=FAIL
+elif denied canary-bash "$LOGA/hooks.log"; then VA=PASS
 elif is_error "$OUTA"; then VA=ERROR
 else VA=INCONCLUSIVE; fi
 
@@ -195,8 +233,10 @@ for _ in $(seq 1 $ATTEMPTS); do
   read_denied "$LOGB/hooks.log" && break
   printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT" && break
 done
-if read_denied "$LOGB/hooks.log"; then VB=PASS
-elif printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT"; then VB=FAIL
+# Same ordering rule as probe A: the sentinel leaking into the transcript proves
+# the read happened, which outranks our own log claiming we denied it.
+if printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT"; then VB=FAIL
+elif read_denied "$LOGB/hooks.log"; then VB=PASS
 elif is_error "$OUTB"; then VB=ERROR
 else VB=INCONCLUSIVE; fi
 

package/.next/standalone/integration-suite/report.js

@@ -1,15 +1,23 @@
 #!/usr/bin/env node
 // Build the Slack report from probe verdicts + diff against last run's state.
-// Args: <resultsJson> <statePath> <model>
+// Args: <resultsJson> <statePath> <model> [channel] [peerStatePath]
 // Each result: {cli, probes:{...}, version?, gated?}. Prints report to stdout
 // AND writes new state: clis[cli] = {probes, version}. Flags broke/recovered.
+//
+// On the BETA channel the report answers a different question — not "is
+// enforcement broken?" but "is it about to break?" — so it compares each CLI
+// against the sibling stable leg's last known state (peerStatePath).
 const fs = require("fs");
-const [, , resultsJson, statePath, model] = process.argv;
+const [, , resultsJson, statePath, model, channel = "stable", peerStatePath = "", eligible = ""] = process.argv;
 const results = JSON.parse(resultsJson);
+const isBeta = channel !== "stable";
 
 let prev = { clis: {}, lastRun: null };
 try { prev = JSON.parse(fs.readFileSync(statePath, "utf8")); } catch {}
 
+let peer = { clis: {} };
+if (peerStatePath) { try { peer = JSON.parse(fs.readFileSync(peerStatePath, "utf8")); } catch {} }
+
 const probesOf = (entry) => (entry ? (entry.probes || entry) : null); // compat old flat schema
 const statusOf = (probes) => {
   const v = Object.values(probes || {});
@@ -23,16 +31,41 @@ const emo = { green: "🟢", yellow: "🟡", red: "🔴", grey: "⚪", error: "
 const rank = { green: 0, grey: 1, yellow: 1, error: 1, red: 2 };
 
 const now = new Date().toISOString().replace(/\.\d+Z$/, "Z");
-const lines = [`🧪 *failproofai CLI integration tests*  ·  ${now}  ·  model=${model}`];
+const title = isBeta
+  ? `🔮 *failproofai CLI pre-release watch*  ·  ${now}  ·  channel=${channel}  ·  model=${model}`
+  : `🧪 *failproofai CLI integration tests*  ·  ${now}  ·  model=${model}`;
+const lines = [title];
 let worst = "green";
-const breaks = [], recoveries = [];
+const breaks = [], recoveries = [], incoming = [];
+
+// A beta CLI counts as "incoming breakage" only when the SAME CLI is green on the
+// stable leg — otherwise we would report an already-broken CLI twice — and only
+// after two consecutive non-green probes, so a broken alpha that gets reverted
+// before release doesn't burn anyone's attention. Strike counts live in state.
+const strikesOf = (cli) => (prev.clis && prev.clis[cli] && prev.clis[cli].strikes) || 0;
+const newStrikes = {};
 
 for (const r of results) {
   const st = statusOf(r.probes);
   const prevEntry = prev.clis && prev.clis[r.cli];
   const prevSt = prevEntry ? statusOf(probesOf(prevEntry)) : null;
   let mark = "";
-  if (r.gated) {
+  if (isBeta) {
+    const peerEntry = peer.clis && peer.clis[r.cli];
+    const peerSt = peerEntry ? statusOf(probesOf(peerEntry)) : null;
+    const strikes = st === "green" ? 0 : strikesOf(r.cli) + 1;
+    newStrikes[r.cli] = strikes;
+    if (r.gated) {
+      mark = `  🔵 _gated: pre-release unchanged, last green_`;
+    } else if (st !== "green" && peerSt === "green") {
+      mark = strikes >= 2
+        ? `  🚨 *INCOMING* — green on stable, ${st === "red" ? "FAILING" : "not verifiable"} on ${channel} (${strikes} runs)`
+        : `  👀 _first ${channel} miss; confirming next run_`;
+      if (strikes >= 2) incoming.push(r.cli);
+    } else if (st !== "green" && peerSt && peerSt !== "green") {
+      mark = `  ⏸️ _also ${peerSt} on stable — the stable leg owns this alarm_`;
+    }
+  } else if (r.gated) {
     mark = `  🔵 _gated: CLI v${r.version || "?"} + failproofai ${r.fpSha || "?"} unchanged, last green_`;
   } else if (prevSt && prevSt !== "red" && st === "red") {
     mark = "  ⚠️ *BROKE*"; breaks.push(r.cli);
@@ -48,7 +81,19 @@ for (const r of results) {
 
 const errored = results.filter((r) => statusOf(r.probes) === "error").map((r) => r.cli);
 let hdr;
-if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`;
+if (isBeta) {
+  // Coverage is stated explicitly and always: only some vendors publish a
+  // pre-release ref, and "beta: all green" must never read as assurance for the
+  // CLIs that were never probed at all. The denominator is how many are ELIGIBLE
+  // (passed in by run.sh), not 12 — a targeted run must not imply the CLIs it
+  // skipped have no pre-release ref, and hardcoding the eligible list here would
+  // be a third copy to drift against.
+  const total = Number(eligible) || results.length;
+  const cov = `_watching ${results.length}/${total} CLIs that publish a pre-release ref (of 12 total)_`;
+  hdr = incoming.length
+    ? `🚨 *INCOMING BREAKAGE*: ${incoming.join(", ")} — broken on ${channel}, still green on stable. Fix before it ships.\n${cov}`
+    : `🔮 no incoming breakage detected on ${channel}\n${cov}`;
+} else if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`;
 // Any CLI still red dominates the header — a recovery elsewhere must not mask it.
 else if (worst === "red") hdr = "🔴 still broken (unchanged)";
 else if (recoveries.length) hdr = `✅ recovered: ${recoveries.join(", ")}`;
@@ -66,6 +111,7 @@ for (const r of results) {
     probes: r.probes,
     version: r.version || (prevEntry && prevEntry.version) || null,
     fpSha: r.fpSha || (prevEntry && prevEntry.fpSha) || null,
+    ...(isBeta ? { strikes: newStrikes[r.cli] || 0 } : {}),
   };
 }
 try { fs.writeFileSync(statePath, JSON.stringify(newState, null, 2)); }

package/.next/standalone/integration-suite/run.sh

@@ -30,7 +30,29 @@ ENVFILE="${CANARY_ENVFILE:?CANARY_ENVFILE (docker --env-file with gateway creds)
 MODEL="${CANARY_LLM_MODEL:-deepseek-v4-pro}"
 GATED="${CANARY_VERSION_GATED-all}"
 
-CLIS=("$@"); [ ${#CLIS[@]} -eq 0 ] && CLIS=(claude codex copilot cursor factory devin antigravity goose opencode pi hermes openclaw)
+CHANNEL="${CANARY_CHANNEL:-stable}"
+# Sibling leg's state file, for the cross-leg comparison (see report.js). The beta
+# leg needs to know whether a CLI is green on stable: "stable green + beta not
+# green" is the incoming-breakage signal, and it is the ONLY way to distinguish
+# "the vendor is about to break us" from "this CLI is broken for everyone already".
+PEER_STATE="${CANARY_PEER_STATE:-}"
+
+# The CLIs with a public pre-release ref — kept in sync with the beta refs in
+# install-clis.sh (__tests__/integration-suite/channel-refs.test.ts asserts the two
+# lists agree, since a silent drift here would look like coverage while probing
+# nothing). Its LENGTH is also the honest denominator for the beta report: a
+# targeted run (`run.sh cursor`) must not imply the CLIs it skipped have no
+# pre-release ref.
+BETA_CLIS=(claude codex copilot cursor goose openclaw)
+
+CLIS=("$@")
+if [ ${#CLIS[@]} -eq 0 ]; then
+  if [ "$CHANNEL" = stable ]; then
+    CLIS=(claude codex copilot cursor factory devin antigravity goose opencode pi hermes openclaw)
+  else
+    CLIS=("${BETA_CLIS[@]}")
+  fi
+fi
 
 # failproofai main HEAD — the second gate dimension. Prefer the value the workflow
 # computed; fall back to git in the checkout.
@@ -72,14 +94,33 @@ for cli in "${CLIS[@]}"; do
     vj="$(node -e 'const st=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));const p=st.clis[process.argv[2]];process.stdout.write(JSON.stringify({cli:process.argv[2],probes:p.probes||p,version:p.version,fpSha:p.fpSha,gated:true}))' "$STATE" "$cli")"
   else
     echo ">> probing $cli ..." >&2
-    vj="$(run_probe "$cli" | sed -n 's/^VERDICT_JSON //p' | tail -1)"
+    # Spool to a file rather than a shell variable: a probe's output is a full agent
+    # transcript from up to 6 CLI invocations, and a stuck or noisy client could make it
+    # arbitrarily large. Only the verdict line and a 20-line tail are ever needed.
+    out_file="$(mktemp)"
+    run_probe "$cli" > "$out_file"
+    vj="$(sed -n 's/^VERDICT_JSON //p' "$out_file" | tail -1)"
+    # Echo the probe tail whenever the verdict is not a clean pass. Without this the
+    # ONLY thing that survived a probe was its VERDICT_JSON line, so a yellow/red run
+    # said WHAT broke and never WHY — and re-running told you no more, because the
+    # cause (a vendor error message) was discarded both times. Diagnosing codex's
+    # 0.145.0 regression needed a full local reproduction purely for want of these
+    # lines. Safe in a public log: every credential here is a registered Actions
+    # secret, so GitHub masks it on the way out.
+    case "$vj" in
+      *FAIL*|*INCONCLUSIVE*|*ERROR*|"")
+        echo "── $cli probe output (tail) ──" >&2
+        tail -20 "$out_file" >&2
+        echo "── end $cli ──" >&2 ;;
+    esac
+    rm -f "$out_file"
     [ -z "$vj" ] && vj="{\"cli\":\"$cli\",\"probes\":{}}"
     vj="$(node -e 'const v=JSON.parse(process.argv[1]);v.version=process.argv[2]||null;v.fpSha=process.argv[3]||null;process.stdout.write(JSON.stringify(v))' "$vj" "$cur_ver" "$FP_SHA")"
   fi
   results="$(node -e 'const a=JSON.parse(process.argv[1]);a.push(JSON.parse(process.argv[2]));process.stdout.write(JSON.stringify(a))' "$results" "$vj")"
 done
 
-report="$(node "$HERE/report.js" "$results" "$STATE" "$MODEL")"
+report="$(node "$HERE/report.js" "$results" "$STATE" "$MODEL" "$CHANNEL" "$PEER_STATE" "${#BETA_CLIS[@]}")"
 echo "════ report ════" >&2; printf '%s\n' "$report" >&2; echo "════════════════" >&2
 printf '%s\n' "$report"   # also to stdout for the workflow log / artifact
 
@@ -94,6 +135,17 @@ else
   echo "(no CANARY_SLACK_WEBHOOK set — report not posted)" >&2
 fi
 
+# The beta leg is ADVISORY and must never fail the job. It probes vendor
+# pre-release builds, which are broken-by-nature often enough that gating CI on
+# them would train everyone to ignore a red run — and the thing it reports is
+# "this WILL break on release", not "this IS broken", which is not a reason to
+# stop the world. The report is emitted and posted either way, so the signal is
+# never lost; escalation is the report's job, not the exit code's.
+if [ "$CHANNEL" != stable ]; then
+  echo "(channel=$CHANNEL — advisory only, not failing the job)" >&2
+  exit 0
+fi
+
 # Fail the job when any probe reported a hard FAIL (broken enforcement) so this
 # scheduled check actually blocks regressions. ERROR (vendor quota/auth) and
 # INCONCLUSIVE (model didn't attempt the tool) are NOT failures — they mean

package/.next/standalone/lib/entry-keys.ts

@@ -0,0 +1,36 @@
+/**
+ * Stable per-entry identity for the log viewer.
+ *
+ * Deliberately standalone and dependency-free: `lib/log-entries.ts` reaches
+ * fs/promises through its project-resolution imports, so a *value* import of
+ * it from a client component drags node:fs into the browser bundle and the
+ * session page 500s. Client code may only import types from there.
+ */
+import type { LogEntry } from "./log-entries";
+
+/**
+ * Assigns every entry a unique, render-stable identity.
+ *
+ * Not every CLI writes a per-record `uuid` — Codex, Copilot, Cursor and Pi
+ * transcripts have none, so `baseEntry` leaves it "". Keying rows off
+ * `uuid || timestamp` therefore collapses to the timestamp, which those CLIs
+ * reuse freely (one 771-record Codex session: 93 timestamps shared by 2-5
+ * records each). Duplicate React keys break reconciliation, and in the
+ * virtualized log list that strands orphaned DOM nodes stacked on top of live
+ * rows — they keep their old transform and never recover (#292 follow-up).
+ *
+ * Collisions are disambiguated by occurrence order, so keys stay stable for as
+ * long as the parsed entry list is — which is what React and the virtualizer's
+ * measurement cache both require.
+ */
+export function buildEntryKeys(entries: LogEntry[]): Map<LogEntry, string> {
+  const keys = new Map<LogEntry, string>();
+  const seen = new Map<string, number>();
+  for (const entry of entries) {
+    const base = entry.uuid || `${entry._source}:${entry.timestamp}`;
+    const n = seen.get(base) ?? 0;
+    seen.set(base, n + 1);
+    keys.set(entry, n === 0 ? base : `${base}#${n}`);
+  }
+  return keys;
+}