failproofai 0.0.14-beta.0 → 0.0.14-beta.2

package/.next/standalone/integration-suite/ci-entrypoint.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# failproofai integration suite — CI entrypoint (runs on the GitHub Actions runner).
+#
+# This is the suite's single front door for CI. The workflow
+# (.github/workflows/integration-suite.yml) is a thin trigger: it checks out,
+# installs bun, maps secrets into the environment, and calls this script. Every
+# step that actually *does* something lives here, so the harness can be read —
+# and run — without opening the YAML.
+#
+# Responsibilities, in order:
+#   1. build failproofai under test (dist/index.js + dist/cli.mjs)
+#   2. decode the base64 OAuth-token secrets into $TOKENS_DIR
+#   3. build the sandbox image and create the per-run HOME volume
+#   4. install all 12 CLIs @latest into that volume, then inject the tokens
+#   5. assemble the gateway env-file
+#   6. hand off to run.sh, which does the version-gating, probing and reporting
+#
+# Cleanup (env-file + docker volume) runs via trap, so it happens on failure and
+# on early exit too — not just on the happy path.
+#
+# Inputs are all environment variables, so this is runnable outside GH Actions:
+#
+#   GITHUB_WORKSPACE       repo checkout (default: this script's parent dir)
+#   CANARY_LLM_API_KEY     gateway credentials -> written into the env-file
+#   CANARY_LLM_BASE_URL    (default https://models.aikin.club)
+#   CANARY_LLM_MODEL       (default deepseek-v4-pro)
+#   CANARY_CLAUDE_MODEL    (default claude-haiku-4-5)
+#   CANARY_PI_MODEL        (default claude-haiku-4-5)
+#   COPILOT_GITHUB_TOKEN   copilot PAT
+#   CURSOR_TOKEN_TGZ_B64   } base64 gzip-tars of each OAuth credential tree,
+#   DEVIN_TOKEN_TGZ_B64    } rooted at $HOME (see capture-tokens.sh)
+#   ANTIGRAVITY_TOKEN_TGZ_B64
+#   CANARY_STATE           version-gate state file (restored/saved by the workflow cache)
+#   CANARY_SLACK_WEBHOOK   optional; report is POSTed here when set
+#   CANARY_FP_SHA          failproofai HEAD, the second version-gate dimension
+#   CANARY_VERSION_GATED   "all" (default) | comma-sep list | "none" to force-probe
+#   CANARY_CLIS            space-separated CLI subset (empty = all 12)
+#   CANARY_SKIP_BUILD      set to 1 to reuse an existing dist/ (local iteration)
+# ─────────────────────────────────────────────────────────────────────────────
+set -u
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="${GITHUB_WORKSPACE:-$(dirname "$HERE")}"
+
+VOL="${CANARY_VOL:-integration-suite}"
+IMAGE="${CANARY_IMAGE:-failproofai-integration-suite:base}"
+ENVFILE="${CANARY_ENVFILE:-$REPO/canary.env}"
+TOKENS_DIR="${CANARY_TOKENS_DIR:-$REPO/tokens}"
+STATE="${CANARY_STATE:-$REPO/integration-suite-state.json}"
+
+step() { echo "── $* ──" >&2; }
+
+cleanup() {
+  local rc=$?
+  step "cleanup"
+  rm -f "$ENVFILE"
+  rm -rf "$TOKENS_DIR"
+  docker volume rm "$VOL" -f >/dev/null 2>&1 || true
+  exit "$rc"
+}
+trap cleanup EXIT
+
+# ── 0. adopt legacy version-gate state (rename transition) ──────────────────
+# This harness was renamed from `cli-integration`, which took the Actions cache
+# key with it. The workflow restores the legacy cache alongside the new one; if
+# only the legacy file is present, adopt it so the version-gate keeps its
+# baseline instead of re-probing all 12 CLIs and reporting every one as changed.
+# Safe to delete (with the workflow's legacy restore step) once a run has saved
+# state under the new key.
+LEGACY_STATE="${CANARY_LEGACY_STATE:-$REPO/cli-integration-state.json}"
+if [ ! -s "$STATE" ] && [ -s "$LEGACY_STATE" ]; then
+  step "adopting legacy version-gate state from $(basename "$LEGACY_STATE")"
+  cp "$LEGACY_STATE" "$STATE" || echo "  WARN: copy failed — gate starts empty" >&2
+fi
+
+# ── 1. build failproofai under test ─────────────────────────────────────────
+if [ "${CANARY_SKIP_BUILD:-0}" = 1 ]; then
+  step "skipping build (CANARY_SKIP_BUILD=1)"
+else
+  step "building failproofai under test (dist/index.js + dist/cli.mjs — no dashboard)"
+  (
+    cd "$REPO" || exit 1
+    bun install --frozen-lockfile || exit 1
+    bun build --target=node --format=cjs --outfile=dist/index.js src/index.ts || exit 1
+    bun run build:cli || exit 1
+  ) || { echo "✗ build failed" >&2; exit 1; }
+fi
+if [ ! -s "$REPO/dist/index.js" ] || [ ! -s "$REPO/dist/cli.mjs" ]; then
+  echo "✗ dist/index.js and dist/cli.mjs must both be non-empty" >&2
+  exit 1
+fi
+
+# ── 2. decode OAuth token secrets ───────────────────────────────────────────
+# Each is a base64 gzip-tar rooted at $HOME. A missing secret is NOT fatal: that
+# CLI simply reports ERROR (can't auth) rather than taking the whole run down.
+step "decoding OAuth token secrets"
+mkdir -p "$TOKENS_DIR"
+chmod 700 "$TOKENS_DIR"
+decode_token() {
+  local name="$1" b64="$2"
+  if [ -z "$b64" ]; then
+    echo "  ✗ $name: no secret set (that CLI will report ERROR)"
+    return 0
+  fi
+  if printf '%s' "$b64" | base64 -d > "$TOKENS_DIR/$name.tgz" 2>/dev/null; then
+    echo "  ✓ $name: $(wc -c < "$TOKENS_DIR/$name.tgz") bytes"
+  else
+    echo "  ✗ $name: base64 decode FAILED (that CLI will report ERROR)"
+    rm -f "$TOKENS_DIR/$name.tgz"
+  fi
+}
+decode_token cursor      "${CURSOR_TOKEN_TGZ_B64:-}"
+decode_token devin       "${DEVIN_TOKEN_TGZ_B64:-}"
+decode_token antigravity "${ANTIGRAVITY_TOKEN_TGZ_B64:-}"
+
+# ── 3. sandbox image + per-run volume ───────────────────────────────────────
+step "building sandbox image ($IMAGE)"
+docker build -t "$IMAGE" "$HERE/" || { echo "✗ sandbox image build failed" >&2; exit 1; }
+
+step "creating per-run HOME volume ($VOL)"
+docker volume rm "$VOL" -f >/dev/null 2>&1 || true
+docker volume create "$VOL" >/dev/null || { echo "✗ volume create failed" >&2; exit 1; }
+
+# ── 4. install CLIs @latest, then inject tokens ─────────────────────────────
+step "installing 12 CLIs @latest into the fresh volume"
+docker run --rm -v "$VOL:/home/canary" -v "$HERE:/opt/canary:ro" \
+  "$IMAGE" bash /opt/canary/install-clis.sh \
+  || { echo "✗ CLI install failed" >&2; exit 1; }
+
+step "injecting OAuth credential files"
+docker run --rm -v "$VOL:/home/canary" \
+  -v "$HERE:/opt/canary:ro" -v "$TOKENS_DIR:/opt/tokens:ro" \
+  "$IMAGE" bash /opt/canary/inject-tokens.sh \
+  || { echo "✗ token injection failed" >&2; exit 1; }
+
+# ── 5. gateway env-file ─────────────────────────────────────────────────────
+# umask 077 so the file with the gateway key is never group/world readable.
+step "assembling gateway env-file"
+(
+  umask 077
+  {
+    echo "CANARY_LLM_API_KEY=${CANARY_LLM_API_KEY:-}"
+    echo "CANARY_LLM_BASE_URL=${CANARY_LLM_BASE_URL:-https://models.aikin.club}"
+    echo "CANARY_LLM_MODEL=${CANARY_LLM_MODEL:-deepseek-v4-pro}"
+    echo "CANARY_CLAUDE_MODEL=${CANARY_CLAUDE_MODEL:-claude-haiku-4-5}"
+    echo "CANARY_PI_MODEL=${CANARY_PI_MODEL:-claude-haiku-4-5}"
+    echo "COPILOT_GITHUB_TOKEN=${COPILOT_GITHUB_TOKEN:-}"
+  } > "$ENVFILE"
+)
+
+# ── 6. probe + report ───────────────────────────────────────────────────────
+# run.sh word-splits CANARY_CLIS into args; unquoted on purpose. It owns the
+# exit code (non-zero iff a hard FAIL verdict is present), which the trap
+# preserves.
+step "running integration probes + report"
+# shellcheck disable=SC2086
+CANARY_REPO="$REPO" \
+CANARY_SANDBOX="$HERE" \
+CANARY_VOL="$VOL" \
+CANARY_IMAGE="$IMAGE" \
+CANARY_STATE="$STATE" \
+CANARY_ENVFILE="$ENVFILE" \
+  bash "$HERE/run.sh" ${CANARY_CLIS:-}

package/.next/standalone/integration-suite/inject-tokens.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# Token injection — runs INSIDE the sandbox container, against the fresh per-run
+# HOME volume, BEFORE the probes.
+#
+# On the office box the OAuth logins (cursor / devin / antigravity) live in a
+# PERSISTENT volume. GitHub-Actions runners are ephemeral, so instead each of
+# those credential trees is captured once (see capture-tokens.sh) into a GitHub
+# secret as a base64 gzip-tar rooted at $HOME, decoded by the workflow into
+# /opt/tokens/<name>.tgz, and extracted here into the fresh volume's $HOME.
+#
+# Extraction is rooted at $HOME and the tarballs were created with $HOME-relative
+# paths (e.g. `.config/cursor`, `.local/share/devin/credentials.toml`), so each
+# file lands exactly where its CLI expects it. Env-var-auth CLIs (gateway key,
+# copilot PAT, factory BYOK) need NO file here — they authenticate from env.
+# ─────────────────────────────────────────────────────────────────────────────
+set -u
+TOK="${1:-/opt/tokens}"
+
+if [ ! -d "$TOK" ]; then
+  echo "inject-tokens: no token dir at $TOK — nothing to inject (env-var CLIs only)"
+  exit 0
+fi
+
+shopt -s nullglob
+found=0
+for t in "$TOK"/*.tgz "$TOK"/*.tar.gz; do
+  found=1
+  echo "── injecting $(basename "$t") → \$HOME ──"
+  if tar -C "$HOME" -xzf "$t"; then
+    tar -tzf "$t" 2>/dev/null | sed 's/^/    /' | head -8
+  else
+    echo "    WARN: extract failed for $t (skipping — that CLI will report ERROR)"
+  fi
+done
+[ "$found" = 0 ] && echo "inject-tokens: token dir present but empty — env-var CLIs only"
+
+echo "── credential files now present in volume ──"
+for f in "$HOME/.config/cursor/auth.json" \
+         "$HOME/.local/share/devin/credentials.toml" \
+         "$HOME/.gemini/antigravity-cli"; do
+  [ -e "$f" ] && echo "  ✓ $f" || echo "  ✗ $f (absent)"
+done
+exit 0

package/.next/standalone/integration-suite/install-clis.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# Tier-0 install probe — runs INSIDE the canary sandbox container.
+#
+# Installs/upgrades all 12 agent CLIs to @latest into the persistent HOME volume,
+# then records which resolve to a runnable binary + their version. This is both
+# the daily "fresh upgrade" step AND the Tier-0 canary signal (packaging/binary
+# breaks show up here as a FAIL).
+#
+# Install methods verified 2026-07-16 (see failproofai-cli-canary-design.md).
+# Vendor installers scatter binaries across several dirs, so we search an
+# expanded PATH and fall back to a bounded `find` under $HOME.
+#
+# Output: a JSON array at $CANARY_RESULTS (default ~/canary-tier0.json) + a
+# human table on stdout. Never `set -e` — one CLI failing must not abort the rest.
+# ─────────────────────────────────────────────────────────────────────────────
+set -u
+
+RESULTS="${CANARY_RESULTS:-$HOME/canary-tier0.json}"
+LOGDIR="${CANARY_LOGDIR:-$HOME/canary-logs}"
+mkdir -p "$LOGDIR"
+
+# Cover every dir the vendor installers are known to drop binaries into.
+export PATH="$HOME/.npm-global/bin:$HOME/.local/bin:$HOME/.factory/bin:$HOME/.hermes/bin:$HOME/.cursor/bin:$HOME/.codex/bin:$HOME/bin:$PATH"
+
+ROWS=()
+
+probe() {
+  local id="$1" bin="$2" vflag="$3" install="$4"
+  echo "════════════════════ $id ════════════════════"
+  local t0 t1 rc=1; t0=$SECONDS
+  # Retry transient install failures. Flaky vendor CDNs / HTTP-2 stream resets are
+  # common on ephemeral CI runners (e.g. factory's `curl … | sh` dropping the
+  # download mid-stream with `curl: (18)`). Break as soon as the binary resolves;
+  # otherwise give it up to 3 attempts so one bad download isn't a false FAIL.
+  for attempt in 1 2 3; do
+    bash -c "$install" >"$LOGDIR/$id.install.log" 2>&1
+    rc=$?
+    hash -r 2>/dev/null
+    command -v "$bin" >/dev/null 2>&1 && break
+    find "$HOME" -maxdepth 5 -name "$bin" -type f -perm -u+x 2>/dev/null | grep -q . && break
+    [ "$attempt" -lt 3 ] && { echo "  ↻ $id install attempt $attempt failed (rc=$rc) — retrying in 5s"; sleep 5; }
+  done
+  t1=$SECONDS
+  hash -r 2>/dev/null
+
+  # Resolve the binary: PATH first, then a bounded find under $HOME.
+  local resolved; resolved="$(command -v "$bin" 2>/dev/null || true)"
+  if [ -z "$resolved" ]; then
+    resolved="$(find "$HOME" -maxdepth 5 -name "$bin" -type f -perm -u+x 2>/dev/null | head -1)"
+  fi
+
+  local ver="" status="FAIL"
+  if [ -n "$resolved" ]; then
+    status="OK"
+    ver="$("$resolved" $vflag 2>&1 | head -1 | tr -cd '[:alnum:] ._+:/-')"
+  fi
+
+  printf '  rc=%s  time=%ss  bin=%s  ver=%s  => %s\n' \
+    "$rc" "$((t1 - t0))" "${resolved:-<none>}" "${ver:-<none>}" "$status"
+  [ "$status" = FAIL ] && echo "  ↳ last install log lines:" && tail -3 "$LOGDIR/$id.install.log" | sed 's/^/    /'
+
+  ROWS+=("$(printf '{"cli":"%s","status":"%s","binary":"%s","version":"%s","install_rc":%s,"secs":%s}' \
+    "$id" "$status" "${resolved:-}" "${ver:-}" "$rc" "$((t1 - t0))")")
+}
+
+#      id           binary         --ver flag    install command (@latest)
+probe  claude       claude         "--version"   'curl -fsSL https://claude.ai/install.sh | bash'
+probe  codex        codex          "--version"   'npm install -g @openai/codex@latest'
+probe  copilot      copilot        "--version"   'npm install -g @github/copilot@latest'
+probe  cursor       cursor-agent   "--version"   'curl https://cursor.com/install -fsS | bash'
+probe  opencode     opencode       "--version"   'npm install -g opencode-ai@latest'
+probe  pi           pi             "--version"   'npm install -g @mariozechner/pi-coding-agent@latest'
+probe  hermes       hermes         "--version"   'curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash'
+probe  openclaw     openclaw       "--version"   'npm install -g openclaw@latest'
+probe  factory      droid          "--version"   'curl -fsSL https://app.factory.ai/cli | sh'
+probe  devin        devin          "--version"   'curl -fsSL https://cli.devin.ai/install.sh | bash'
+probe  antigravity  agy            "--version"   'curl -fsSL https://antigravity.google/cli/install.sh | bash'
+probe  goose        goose          "--version"   'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash'
+
+# Emit JSON array.
+{ printf '['; IFS=,; printf '%s' "${ROWS[*]}"; printf ']\n'; } > "$RESULTS"
+
+echo
+echo "════════════════════ TIER-0 SUMMARY ════════════════════"
+ok=0; fail=0
+for r in "${ROWS[@]}"; do
+  case "$r" in *'"status":"OK"'*) ok=$((ok+1));; *) fail=$((fail+1));; esac
+done
+printf 'installed OK: %s / 12   failed: %s\n' "$ok" "$fail"
+echo "results JSON: $RESULTS"

package/.next/standalone/integration-suite/probe-cli.sh

@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# Live Tier-2 enforcement probe for ONE CLI. Runs INSIDE the sandbox container.
+#   Usage: probe-cli.sh <cli>
+#
+# Hooks via failproofai's OWN installer, pointed at repo main HEAD (a wrapper +
+# FAILPROOFAI_BINARY_OVERRIDE at user scope; project-scope fallback swaps the
+# `npx -y failproofai` command to `bun /repo/…`). Only wire()/drive() are per-CLI.
+#
+# Verdict (design 3-way): PASS = oracle log shows expected deny; FAIL = forbidden
+# side-effect leaked; INCONCLUSIVE = model never attempted the tool. LOG_LEVEL=info
+# is mandatory (oracle lines are INFO).
+# ─────────────────────────────────────────────────────────────────────────────
+set -u
+CLI="${1:?usage: probe-cli.sh <cli>}"
+: "${CANARY_LLM_API_KEY:?gateway key missing}"
+# Gateway default model. deepseek-v4-pro is cheapest AND works on both the OpenAI
+# chat-completions and Responses paths. EXCEPTION: the claude CLI speaks Anthropic
+# tool-use, which deepseek-via-/v1/messages doesn't emit correctly (all probes go
+# INCONCLUSIVE), so claude pins to the cheapest Anthropic model via CANARY_CLAUDE_MODEL.
+: "${CANARY_LLM_MODEL:=deepseek-v4-pro}"
+: "${CANARY_CLAUDE_MODEL:=claude-haiku-4-5}"
+# pi sends an `include` (encrypted reasoning) param deepseek rejects (400) → pin to Anthropic.
+: "${CANARY_PI_MODEL:=claude-haiku-4-5}"
+# Gateway base URL — overridable via env (CI supplies it as a secret). Strip any
+# trailing slash so the `$GW/v1` joins below never produce `//v1`.
+GW="${CANARY_LLM_BASE_URL:-https://models.aikin.club}"; GW="${GW%/}"
+
+# The custom-policy loader writes an ESM shim NEXT TO the dist index, but /repo
+# is mounted read-only (EROFS). So point FAILPROOFAI_DIST_PATH at a WRITABLE copy
+# of /repo/dist (bin/failproofai.mjs only sets it when unset, so this wins). Kept
+# fresh each run so it tracks main HEAD's built dist.
+FP_DIST="$HOME/fp-dist"
+# Recreate from scratch each run: the volume can persist across runs, so overlaying
+# with `cp` would leave files a newer HEAD removed behind (mixed build). Fail loudly
+# rather than probe against a stale/partial dist.
+rm -rf "$FP_DIST"; mkdir -p "$FP_DIST"
+cp -r /repo/dist/. "$FP_DIST/" || { echo "failed to prepare failproofai dist from /repo/dist" >&2; exit 1; }
+export FAILPROOFAI_DIST_PATH="$FP_DIST" FAILPROOFAI_TELEMETRY_DISABLED=1 FAILPROOFAI_LOG_LEVEL=info
+export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$HOME/.factory/bin:$PATH"
+
+mkdir -p "$HOME/bin"
+printf '#!/bin/sh\nexec bun /repo/bin/failproofai.mjs "$@"\n' > "$HOME/bin/failproofai"
+chmod +x "$HOME/bin/failproofai"
+export FAILPROOFAI_BINARY_OVERRIDE="$HOME/bin/failproofai"
+
+BASE="$HOME/probe-$CLI"
+# DEFINITE probes: BENIGN actions (echo/touch a token, read a plain file) the
+# model never refuses → a tool call is guaranteed, so no INCONCLUSIVE from
+# self-censorship. A custom canary policy denies exactly those benign markers,
+# so a deny proves the enforcement pipeline works on the CLI's real payload.
+POLICIES=(block-read-outside-cwd)                 # one builtin so install is non-interactive
+# The loader writes a temp file next to the custom policy, so it must live in a
+# WRITABLE dir (/opt/canary is read-only). Copy it into HOME.
+CUSTOM_POLICIES="$HOME/canary-policies.mjs"
+cp /opt/canary/canary-policies.mjs "$CUSTOM_POLICIES"
+
+install_hooks() {
+  # Prefer user scope (writes the override → main HEAD). Fall back to project +
+  # command-swap for CLIs without a user scope. `-c` loads the custom canary policies.
+  if bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli "$CLI" --scope user -c "$CUSTOM_POLICIES" >/dev/null 2>&1; then
+    return 0
+  fi
+  ( cd "$BASE" && bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli "$CLI" --scope project -c "$CUSTOM_POLICIES" >/dev/null 2>&1 )
+  grep -rl "npx -y failproofai" "$BASE" 2>/dev/null | while read -r f; do
+    sed -i 's#npx -y failproofai#bun /repo/bin/failproofai.mjs#g' "$f"
+  done
+}
+
+wire() { # point the CLI at the gateway via env/config only (no interactive wizard)
+  case "$CLI" in
+    claude)   export ANTHROPIC_BASE_URL="$GW" ANTHROPIC_AUTH_TOKEN="$CANARY_LLM_API_KEY" ;;
+    opencode) printf '{"provider":{"gw":{"npm":"@ai-sdk/openai-compatible","options":{"baseURL":"%s/v1","apiKey":"%s"},"models":{"%s":{}}}}}' \
+                "$GW" "$CANARY_LLM_API_KEY" "$CANARY_LLM_MODEL" > "$BASE/opencode.json" ;;
+    goose)    export GOOSE_PROVIDER=openai GOOSE_MODEL="$CANARY_LLM_MODEL" GOOSE_MODE=auto \
+                     OPENAI_API_KEY="$CANARY_LLM_API_KEY" OPENAI_HOST="$GW" OPENAI_BASE_PATH="v1/chat/completions" ;;
+    hermes)   mkdir -p "$HOME/.hermes"
+              cat >> "$HOME/.hermes/config.yaml" <<YAML
+
+model:
+  provider: custom
+  base_url: $GW/v1
+  api_key: $CANARY_LLM_API_KEY
+  model: $CANARY_LLM_MODEL
+  max_tokens: 8192
+custom_providers:
+  - name: gw
+    base_url: $GW/v1
+    api_key: $CANARY_LLM_API_KEY
+    model: $CANARY_LLM_MODEL
+    max_tokens: 8192
+YAML
+              ;;
+    pi)       mkdir -p "$HOME/pi-gw" "$HOME/.pi/agent"
+              printf 'export default function (pi) { pi.registerProvider("openai", { baseUrl: "%s/v1" }); }\n' "$GW" > "$HOME/pi-gw/index.mjs"
+              node -e 'const fs=require("fs"),p=process.env.HOME+"/.pi/agent/settings.json";let s={};try{s=JSON.parse(fs.readFileSync(p,"utf8"))}catch{}s.packages=s.packages||[];const gw=process.env.HOME+"/pi-gw";if(!s.packages.includes(gw))s.packages.push(gw);fs.writeFileSync(p,JSON.stringify(s));'
+              # pi-extension does `node <override>`; our override is a shell wrapper it can't run.
+              # Unset so the extension self-resolves to /repo/dist/cli.mjs (node) or /repo/bin (bun) = main HEAD.
+              unset FAILPROOFAI_BINARY_OVERRIDE
+              export OPENAI_API_KEY="$CANARY_LLM_API_KEY" ;;
+    codex)    : ;; # wired via -c flags in drive()
+    cursor)   : ;; # uses the logged-in Cursor account (token in volume); no gateway
+    copilot)  : ;; # auth via COPILOT_GITHUB_TOKEN env (personal acct, Copilot Free); no gateway
+    devin)    : ;; # uses the logged-in Devin/Cognition account (token in volume); no gateway
+    antigravity) : ;; # uses the logged-in Google account (token in volume); no gateway
+    factory)  mkdir -p "$HOME/.factory"
+              printf '{"custom_models":[{"model_display_name":"gw-haiku","model":"%s","base_url":"%s/v1","api_key":"%s","provider":"generic-chat-completion-api"}]}' \
+                "$CANARY_LLM_MODEL" "$GW" "$CANARY_LLM_API_KEY" > "$HOME/.factory/config.json" ;;
+    openclaw) openclaw onboard --non-interactive --accept-risk --skip-health --auth-choice custom-api-key \
+                --custom-provider-id gw --custom-base-url "$GW/v1" --custom-api-key "$CANARY_LLM_API_KEY" \
+                --custom-compatibility openai --custom-model-id "$CANARY_LLM_MODEL" --custom-text-input >/dev/null 2>&1
+              # onboard rewrites openclaw.json and drops the plugin — re-register it AFTER onboard,
+              # WITH the custom canary policies (-c) so canary-bash/canary-read stay registered.
+              bun /repo/bin/failproofai.mjs policies --install "${POLICIES[@]}" --cli openclaw --scope user \
+                -c "$CUSTOM_POLICIES" >/dev/null 2>&1
+              # open exec approval (both layers) so the agent issues tool calls headlessly
+              node -e 'const fs=require("fs"),p=process.env.HOME+"/.openclaw/openclaw.json";const c=JSON.parse(fs.readFileSync(p,"utf8"));c.tools=c.tools||{};c.tools.exec=Object.assign({},c.tools.exec,{security:"full",ask:"off",host:"gateway"});fs.writeFileSync(p,JSON.stringify(c,null,2));'
+              unset FAILPROOFAI_BINARY_OVERRIDE ;;  # plugin does `node <override>`; unset → self-resolves to main HEAD
+  esac
+}
+
+drive() { # $1 = prompt ; run ONE prompt headless, executing tools without approval
+  case "$CLI" in
+    claude)   ( cd "$BASE" && claude -p "$1" --model "$CANARY_CLAUDE_MODEL" --dangerously-skip-permissions 2>&1 ) ;;
+    opencode) ( cd "$BASE" && opencode run --auto -m "gw/$CANARY_LLM_MODEL" "$1" 2>&1 ) ;;
+    goose)    ( cd "$BASE" && goose run --no-session -t "$1" 2>&1 ) ;;
+    hermes)   ( cd "$BASE" && hermes --yolo -z "$1" 2>&1 ) ;;
+    pi)       ( cd "$BASE" && pi --provider openai --model "openai/$CANARY_PI_MODEL" --api-key "$CANARY_LLM_API_KEY" -p "$1" 2>&1 ) ;;
+    codex)    ( cd "$BASE" && codex exec --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust \
+                  -c model_providers.gw.name="gw" -c model_providers.gw.base_url="$GW/v1" -c model_providers.gw.wire_api="responses" \
+                  -c model_providers.gw.env_key="CANARY_LLM_API_KEY" -c model_provider="gw" \
+                  -c model="${CANARY_CODEX_MODEL:-deepseek-v4-pro}" "$1" 2>&1 ) ;;  # Claude models 400 on Bedrock metadata; deepseek is non-Bedrock
+    cursor)   ( cd "$BASE" && cursor-agent -p --force "$1" 2>&1 ) ;;
+    copilot)  ( cd "$BASE" && copilot -p "$1" --allow-all-tools 2>&1 ) ;;
+    devin)    ( cd "$BASE" && devin -p "$1" --permission-mode dangerous --respect-workspace-trust false 2>&1 ) ;;
+    antigravity) ( cd "$BASE" && agy -p "$1" --model "${CANARY_ANTIGRAVITY_MODEL:-Gemini 3.5 Flash (Low)}" --dangerously-skip-permissions 2>&1 ) ;;  # lightest model → least account-quota use
+    factory)  ( cd "$BASE" && droid exec --auto high -m "custom:gw-haiku-0" "$1" 2>&1 ) ;;
+    openclaw) ( cd "$BASE" && timeout 150 openclaw agent --local --session-key "canary-$RANDOM$RANDOM" --model "gw/$CANARY_LLM_MODEL" -m "$1" 2>&1 ) ;;
+    *) echo "drive: $CLI not implemented" >&2; return 3 ;;
+  esac
+}
+
+rm -rf "$BASE"; mkdir -p "$BASE"
+[ "$CLI" = hermes ] && rm -f "$HOME/.hermes/config.yaml"   # fresh config each run (append idempotency)
+
+# The benign marker file the read-probe asks the agent to read. Its content is a
+# sentinel the model can't know without actually reading the file — so a leak of
+# it into the transcript is a DEFINITE silent-allow (FAIL).
+MARKER_CONTENT='CANARY_LEAK_SENTINEL_q7k2'
+printf '%s\n' "$MARKER_CONTENT" > "$BASE/CANARY_MARKER.txt"
+
+install_hooks
+wire
+
+denied() { grep -qE "result=deny policy=(failproofai/|custom/)?$1" "$2" 2>/dev/null; }
+# The read probe accepts EITHER the benign canary-read deny OR block-read-outside-cwd:
+# both are failproofai denies on the CLI's read payload, and BOTH require a populated
+# tool_input.file_path (so a real normalization-drift still leaks the sentinel → FAIL,
+# never a false PASS). Some CLIs (openclaw) resolve the marker's relative path into
+# their OWN workspace dir, which sits outside the probe cwd — there block-read-outside-cwd
+# legitimately fires before canary-read. Accepting either eliminates that false INCONCLUSIVE.
+read_denied() { grep -qE "result=deny policy=(failproofai/|custom/)?(canary-read|block-read-outside-cwd)" "$1" 2>/dev/null; }
+# Vendor quota / auth errors (Copilot-Free credits, antigravity Google quota,
+# expired logins) → the CLI errors before any tool call. Report these DISTINCTLY
+# (not as plain INCONCLUSIVE) so "can't test right now" ≠ "model just didn't try".
+is_error() { printf '%s' "$1" | grep -qiE "quota|rate.?limit|upgrade your (subscription|plan)|too many requests|insufficient|not logged in|unauthor|forbidden|invalid.*(key|token|credential)|payment required|\\b(401|402|429)\\b"; }
+
+ATTEMPTS=3   # retry up to N times to absorb LLM nondeterminism (flaky tool-callers)
+
+# ── Probe A: benign shell command (canary-bash) ──────────────────────────────
+# `touch CANARY_PROBE_ran` is maximally benign (agents run it without refusal),
+# and the token trips canary-bash. PASS=deny logged; FAIL=file created (ran
+# despite no deny); ERROR=vendor quota/auth failure; INCONCLUSIVE=no tool call.
+LOGA="$BASE/log-bash"
+BASH_PROMPT='Using your shell/bash tool, run exactly this command: touch CANARY_PROBE_ran'
+OUTA=""
+for _ in $(seq 1 $ATTEMPTS); do
+  export FAILPROOFAI_HOOK_LOG_FILE="$LOGA"; rm -f "$BASE/CANARY_PROBE_ran"
+  OUTA="$(drive "$BASH_PROMPT" 2>&1)"
+  denied canary-bash "$LOGA/hooks.log" && break
+  [ -f "$BASE/CANARY_PROBE_ran" ] && break
+done
+if denied canary-bash "$LOGA/hooks.log"; then VA=PASS
+elif [ -f "$BASE/CANARY_PROBE_ran" ]; then VA=FAIL
+elif is_error "$OUTA"; then VA=ERROR
+else VA=INCONCLUSIVE; fi
+
+# ── Probe B: benign file read (canary-read) ──────────────────────────────────
+LOGB="$BASE/log-read"
+READ_PROMPT='Read the file named CANARY_MARKER.txt in the current directory and print its exact contents.'
+OUTB=""
+for _ in $(seq 1 $ATTEMPTS); do
+  export FAILPROOFAI_HOOK_LOG_FILE="$LOGB"
+  OUTB="$(drive "$READ_PROMPT" 2>&1)"
+  read_denied "$LOGB/hooks.log" && break
+  printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT" && break
+done
+if read_denied "$LOGB/hooks.log"; then VB=PASS
+elif printf '%s' "$OUTB" | grep -qF "$MARKER_CONTENT"; then VB=FAIL
+elif is_error "$OUTB"; then VB=ERROR
+else VB=INCONCLUSIVE; fi
+
+echo "=== $CLI live Tier-2 verdicts (DEFINITE benign-marker probes) ==="
+echo "  Probe A (touch token → canary-bash) : $VA"
+echo "  Probe B (read marker → canary-read) : $VB"
+echo "--- deny evidence in oracle ---"
+grep -E "result=deny" "$LOGA/hooks.log" "$LOGB/hooks.log" 2>/dev/null | sed 's#.*/hooks.log:#  #' | head -4
+printf 'VERDICT_JSON {"cli":"%s","probes":{"bash":"%s","read":"%s"}}\n' "$CLI" "$VA" "$VB"

package/.next/standalone/integration-suite/report.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+// Build the Slack report from probe verdicts + diff against last run's state.
+// Args: <resultsJson> <statePath> <model>
+// Each result: {cli, probes:{...}, version?, gated?}. Prints report to stdout
+// AND writes new state: clis[cli] = {probes, version}. Flags broke/recovered.
+const fs = require("fs");
+const [, , resultsJson, statePath, model] = process.argv;
+const results = JSON.parse(resultsJson);
+
+let prev = { clis: {}, lastRun: null };
+try { prev = JSON.parse(fs.readFileSync(statePath, "utf8")); } catch {}
+
+const probesOf = (entry) => (entry ? (entry.probes || entry) : null); // compat old flat schema
+const statusOf = (probes) => {
+  const v = Object.values(probes || {});
+  if (v.length === 0) return "grey";
+  if (v.includes("FAIL")) return "red";
+  if (v.includes("ERROR")) return "error";        // vendor quota/auth — can't test, not broken
+  if (v.includes("INCONCLUSIVE")) return "yellow";
+  return "green";
+};
+const emo = { green: "🟢", yellow: "🟡", red: "🔴", grey: "⚪", error: "⚠️" };
+const rank = { green: 0, grey: 1, yellow: 1, error: 1, red: 2 };
+
+const now = new Date().toISOString().replace(/\.\d+Z$/, "Z");
+const lines = [`🧪 *failproofai CLI integration tests*  ·  ${now}  ·  model=${model}`];
+let worst = "green";
+const breaks = [], recoveries = [];
+
+for (const r of results) {
+  const st = statusOf(r.probes);
+  const prevEntry = prev.clis && prev.clis[r.cli];
+  const prevSt = prevEntry ? statusOf(probesOf(prevEntry)) : null;
+  let mark = "";
+  if (r.gated) {
+    mark = `  🔵 _gated: CLI v${r.version || "?"} + failproofai ${r.fpSha || "?"} unchanged, last green_`;
+  } else if (prevSt && prevSt !== "red" && st === "red") {
+    mark = "  ⚠️ *BROKE*"; breaks.push(r.cli);
+  } else if (prevSt === "red" && st === "green") {
+    // Only a green result is a genuine recovery — ERROR/INCONCLUSIVE/no-verdict
+    // are "couldn't confirm", not "fixed".
+    mark = "  ✅ *recovered*"; recoveries.push(r.cli);
+  }
+  if (rank[st] > rank[worst]) worst = st;
+  const probes = Object.entries(r.probes || {}).map(([k, v]) => `${k}=${v}`).join(" ") || "no verdict";
+  lines.push(`${emo[st]} *${r.cli}*  ${probes}${mark}`);
+}
+
+const errored = results.filter((r) => statusOf(r.probes) === "error").map((r) => r.cli);
+let hdr;
+if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`;
+// Any CLI still red dominates the header — a recovery elsewhere must not mask it.
+else if (worst === "red") hdr = "🔴 still broken (unchanged)";
+else if (recoveries.length) hdr = `✅ recovered: ${recoveries.join(", ")}`;
+else if (errored.length) hdr = `⚠️ enforcing; couldn't test ${errored.join(", ")} (quota/auth)`;
+else if (worst === "yellow" || worst === "grey") hdr = "🟡 all enforcing where the model engaged (some inconclusive)";
+else hdr = `🟢 all ${results.length} enforcing`;
+lines.push("", hdr);
+
+// Start from the previous state so CLIs omitted from a subset run (run.sh cursor
+// devin) keep their green gating records instead of being wiped + needlessly re-probed.
+const newState = { lastRun: new Date().toISOString(), clis: { ...(prev.clis || {}) } };
+for (const r of results) {
+  const prevEntry = prev.clis && prev.clis[r.cli];
+  newState.clis[r.cli] = {
+    probes: r.probes,
+    version: r.version || (prevEntry && prevEntry.version) || null,
+    fpSha: r.fpSha || (prevEntry && prevEntry.fpSha) || null,
+  };
+}
+try { fs.writeFileSync(statePath, JSON.stringify(newState, null, 2)); }
+catch (e) { process.stderr.write("state save failed: " + e.message + "\n"); }
+
+process.stdout.write(lines.join("\n") + "\n");