@kontourai/flow-agents 2.0.0 → 2.1.0

package/evals/integration/test_command_log_fork_classification.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# test_command_log_fork_classification.sh
+#
+# The verifier must tell a BENIGN concurrent fork apart from real TAMPER, and
+# the repair tool must refuse to touch tamper. This is what prevents an honest
+# parallel-write race from becoming a hard block an agent is tempted to launder.
+#
+#   forked  = two PostToolUse captures share a parent; all hashes self-consistent
+#             and reachable. NON-blocking advisory; records stay trusted.
+#   broken  = content edit (self-hash mismatch) / reorder / deletion / a
+#             non-capture sibling on a shared parent. Hard block (unchanged).
+#
+# Also proves: repair re-linearizes forked→ok, and REFUSES broken (no laundering).
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+export GATE="$ROOT/scripts/hooks/stop-goal-fit.js"
+REPAIR="$ROOT/scripts/repair-command-log.js"
+
+TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
+errors=0
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+
+SD=".flow-agents/s"
+
+# Build a command-log from a spec: JSON array of {cmd,exit,src,parent} where
+# parent is the 0-based index of the entry whose hash is this entry's prevHash
+# (-1 = genesis). Lets us construct linear chains AND forks deterministically.
+build() { # $1=dir $2=spec-json
+  mkdir -p "$1/$SD"
+  DIR="$1" node -e '
+    const fs=require("fs"),crypto=require("crypto"),path=require("path");
+    const g=require(process.env.GATE), GEN=g.CHAIN_GENESIS_VERIFY;
+    const canon=r=>{const k=Object.keys(r).filter(x=>x!=="_chain").sort();const o={};for(const x of k)o[x]=r[x];return JSON.stringify(o);};
+    const H=(p,r)=>crypto.createHash("sha256").update(p+canon(r)).digest("hex");
+    const spec=JSON.parse(process.argv[1]); const hashes=[],lines=[];
+    spec.forEach((s,i)=>{
+      const rec={command:s.cmd,observedResult:s.exit===0?"pass":"fail",exitCode:s.exit,
+        capturedAt:new Date(Date.UTC(2026,0,1,0,0,i)).toISOString(),source:s.src||"postToolUse-capture"};
+      const prev=s.parent===-1?GEN:hashes[s.parent]; const h=H(prev,rec);
+      hashes.push(h); lines.push(JSON.stringify({...rec,_chain:{seq:i,prevHash:prev,hash:h}}));
+    });
+    fs.writeFileSync(path.join(process.env.DIR,".flow-agents/s/command-log.jsonl"),lines.join("\n")+"\n");
+  ' "$2"
+}
+status() { DIR="$1" node -e 'const g=require(process.env.GATE);console.log(g.verifyCommandLogChain(process.env.DIR+"/.flow-agents/s").status)' ; }
+
+# ── 1. linear → ok ────────────────────────────────────────────────────────────
+D="$TMP/linear"; build "$D" '[{"cmd":"a","exit":0,"parent":-1},{"cmd":"b","exit":0,"parent":0}]'
+[ "$(status "$D")" = "ok" ] && _pass "linear chain → ok" || _fail "linear → $(status "$D"), want ok"
+
+# ── 2. concurrent fork (two captures share a parent) → forked ─────────────────
+D="$TMP/fork"; build "$D" '[{"cmd":"a","exit":0,"parent":-1},{"cmd":"b","exit":0,"parent":0},{"cmd":"c","exit":0,"parent":0}]'
+[ "$(status "$D")" = "forked" ] && _pass "concurrent fork → forked (not broken)" || _fail "fork → $(status "$D"), want forked"
+
+# ── 3. content edit (flip exitCode, keep hash) → broken ───────────────────────
+D="$TMP/flip"; build "$D" '[{"cmd":"npm test","exit":0,"parent":-1},{"cmd":"npm run lint","exit":1,"parent":0}]'
+python3 - "$D/$SD/command-log.jsonl" <<'PY'
+import json,sys
+L=open(sys.argv[1]).read().strip().split("\n"); e=json.loads(L[1]); e["exitCode"]=0; e["observedResult"]="pass"
+L[1]=json.dumps(e); open(sys.argv[1],"w").write("\n".join(L)+"\n")
+PY
+[ "$(status "$D")" = "broken" ] && _pass "content edit → broken (tamper, not fork)" || _fail "flip → $(status "$D"), want broken"
+
+# ── 4. reorder → broken ───────────────────────────────────────────────────────
+D="$TMP/reorder"; build "$D" '[{"cmd":"a","exit":0,"parent":-1},{"cmd":"b","exit":0,"parent":0}]'
+python3 - "$D/$SD/command-log.jsonl" <<'PY'
+import sys
+L=open(sys.argv[1]).read().strip().split("\n"); L[0],L[1]=L[1],L[0]; open(sys.argv[1],"w").write("\n".join(L)+"\n")
+PY
+[ "$(status "$D")" = "broken" ] && _pass "reorder → broken" || _fail "reorder → $(status "$D"), want broken"
+
+# ── 5. deleted predecessor → broken ───────────────────────────────────────────
+D="$TMP/delete"; build "$D" '[{"cmd":"a","exit":0,"parent":-1},{"cmd":"b","exit":0,"parent":0}]'
+python3 - "$D/$SD/command-log.jsonl" <<'PY'
+import sys
+L=open(sys.argv[1]).read().strip().split("\n"); open(sys.argv[1],"w").write(L[1]+"\n")
+PY
+[ "$(status "$D")" = "broken" ] && _pass "deleted predecessor → broken" || _fail "delete → $(status "$D"), want broken"
+
+# ── 6. non-capture sibling on a shared parent → broken (not a benign fork) ─────
+D="$TMP/badfork"; build "$D" '[{"cmd":"a","exit":0,"parent":-1},{"cmd":"b","exit":0,"parent":0},{"cmd":"c","exit":0,"parent":0,"src":"manual-inject"}]'
+[ "$(status "$D")" = "broken" ] && _pass "non-capture sibling fork → broken (conservative)" || _fail "badfork → $(status "$D"), want broken"
+
+# ── 7. repair re-linearizes forked → ok; refuses broken ───────────────────────
+D="$TMP/fork2"; build "$D" '[{"cmd":"a","exit":0,"parent":-1},{"cmd":"b","exit":0,"parent":0},{"cmd":"c","exit":0,"parent":0}]'
+node "$REPAIR" "$D/$SD" --reason "test" >/dev/null 2>&1
+[ "$(status "$D")" = "ok" ] && _pass "repair: forked → ok" || _fail "repair forked → $(status "$D"), want ok"
+
+D="$TMP/flip2"; build "$D" '[{"cmd":"x","exit":0,"parent":-1},{"cmd":"y","exit":1,"parent":0}]'
+python3 - "$D/$SD/command-log.jsonl" <<'PY'
+import json,sys
+L=open(sys.argv[1]).read().strip().split("\n"); e=json.loads(L[1]); e["exitCode"]=0
+L[1]=json.dumps(e); open(sys.argv[1],"w").write("\n".join(L)+"\n")
+PY
+before=$(cat "$D/$SD/command-log.jsonl")
+set +e; node "$REPAIR" "$D/$SD" >/dev/null 2>&1; rc=$?; set -e
+after=$(cat "$D/$SD/command-log.jsonl")
+if [ "$rc" -ne 0 ] && [ "$before" = "$after" ]; then _pass "repair: REFUSES broken (exit!=0, log unchanged — no laundering)"; else _fail "repair touched/accepted a broken log (rc=$rc)"; fi
+
+# ── 8. the Stop gate does NOT hard-block a forked log ─────────────────────────
+D="$TMP/gate"; mkdir -p "$D/$SD"
+printf '# Repo\n' > "$D/AGENTS.md"
+printf '%s' '{"schema_version":"1.0","task_slug":"s","status":"delivered","phase":"done","updated_at":"2026-06-23T00:00:00Z","next_action":{"status":"done","summary":"done"}}' > "$D/$SD/state.json"
+cat > "$D/$SD/s--deliver.md" <<'MD'
+# s
+
+branch: main
+status: delivered
+type: deliver
+
+## Definition Of Done
+- [x] tests pass
+
+## Goal Fit Gate
+- [x] acceptance verified
+
+### Verdict: PASS
+MD
+# forked log whose captures are all PASS, so there is no contradiction to flag
+build "$D" '[{"cmd":"npm test","exit":0,"parent":-1},{"cmd":"npm run build","exit":0,"parent":0},{"cmd":"npm run build","exit":0,"parent":0}]'
+printf '%s' '{"schema_version":"1.0","task_slug":"s","verdict":"pass","checks":[{"id":"t","kind":"command","status":"pass","command":"npm test","summary":"ok"}]}' > "$D/$SD/evidence.json"
+set +e
+out=$(FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_BACKSTOP=skip node "$GATE" 2>&1 <<< "{\"hook_event_name\":\"Stop\",\"cwd\":\"$D\"}")
+rc=$?
+set -e
+if [ "$rc" -eq 0 ]; then _pass "gate does NOT hard-block forked log (exit 0)"; else _fail "gate blocked forked log (exit $rc): $out"; fi
+echo "$out" | grep -q "concurrent-capture fork" && _pass "gate emits the concurrent-fork advisory" || _fail "missing fork advisory: $out"
+echo "$out" | grep -q "command-log integrity check FAILED" && _fail "gate wrongly emitted tamper warning for a fork" || _pass "no false tamper warning for a fork"
+
+echo ""
+if [ "$errors" -eq 0 ]; then echo "fork classification tests passed."; exit 0; fi
+echo "fork classification tests FAILED: $errors issue(s)."; exit 1

package/evals/integration/test_kit_identity_trust.sh

@@ -0,0 +1,393 @@
+#!/usr/bin/env bash
+# test_kit_identity_trust.sh — Regression eval for kit identity end-to-end in the trust chain.
+#
+# Proves Fix 1 and Fix 2 from the kit-identity task:
+#
+#  Fix 1 (surfaceCheckFromArtifact reads kit from bundle, never hardcodes "builder"):
+#   1a. KNOWLEDGE-TYPED bundle → kitIdentityFromBundle derives kitId="knowledge", subject="knowledge-kit"
+#   1b. BUILDER-TYPED bundle  → kitIdentityFromBundle derives kitId="builder", subject="builder-kit"
+#   1c. WORKFLOW-ONLY bundle (no kit-typed claim, no current.json) → kitId="unknown", subject="unknown-kit"
+#   1d. record-evidence --surface-trust-json <knowledge-fixture> completes without crash
+#
+#  Fix 2 (route-back guard is FlowDefinition-driven, not hardcoded to builder.build):
+#   2a. builder.build: verification→execution still enforced (identical behavior preserved)
+#   2b. Custom non-builder flow WITH route_back_policy: verification→execution ENFORCED
+#   2c. Custom flow WITHOUT route_back_policy: verification→execution NOT ENFORCED
+#
+# Deterministic, no model spend, self-cleaning.
+# Usage: bash evals/integration/test_kit_identity_trust.sh
+
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT/evals/lib/node.sh"
+
+TMP="$(mktemp -d)"
+errors=0
+
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+
+cleanup() { rm -rf "$TMP"; }
+trap cleanup EXIT
+
+SIDECAR_JS="${ROOT}/build/src/cli/workflow-sidecar.js"
+SIDECAR_BUNDLE_WRITER="workflow-sidecar"
+
+echo ""
+echo "=== Fix 1: kitIdentityFromBundle reads kit from bundle claims (not hardcoded 'builder') ==="
+
+# ─── Write fixture bundle files (note: argv[2] = file path since argv[1] = "-" for stdin) ─────────
+
+node - "$TMP/knowledge.bundle" << 'NODE'
+const fs = require('fs');
+// argv[0]=node, argv[1]="-", argv[2]=file path
+const bundlePath = process.argv[2];
+const bundle = {
+  schemaVersion: 3, source: "test-fixture",
+  claims: [{
+    id: "c-knowledge-1", claimType: "knowledge.verify.tests",
+    subjectType: "flow-step", subjectId: "test-slug/knowledge-ev",
+    surface: "flow-agents.workflow", fieldOrBehavior: "knowledge verification",
+    value: "pass", status: "verified",
+    createdAt: "2026-06-27T00:00:00Z", updatedAt: "2026-06-27T00:00:00Z",
+    impactLevel: "high", verificationPolicyId: "policy:knowledge.verify.tests"
+  }],
+  evidence: [], policies: [], events: []
+};
+fs.writeFileSync(bundlePath, JSON.stringify(bundle, null, 2));
+NODE
+
+node - "$TMP/builder.bundle" << 'NODE'
+const fs = require('fs');
+const bundlePath = process.argv[2];
+const bundle = {
+  schemaVersion: 3, source: "test-fixture",
+  claims: [{
+    id: "c-builder-1", claimType: "builder.verify.tests",
+    subjectType: "flow-step", subjectId: "test-slug/builder-ev",
+    surface: "flow-agents.workflow", fieldOrBehavior: "builder verification",
+    value: "pass", status: "verified",
+    createdAt: "2026-06-27T00:00:00Z", updatedAt: "2026-06-27T00:00:00Z",
+    impactLevel: "high", verificationPolicyId: "policy:builder.verify.tests"
+  }],
+  evidence: [], policies: [], events: []
+};
+fs.writeFileSync(bundlePath, JSON.stringify(bundle, null, 2));
+NODE
+
+node - "$TMP/workflow-only.bundle" << 'NODE'
+const fs = require('fs');
+const bundlePath = process.argv[2];
+const bundle = {
+  schemaVersion: 3, source: "test-fixture",
+  claims: [{
+    id: "c-wf-1", claimType: "workflow.check.build",
+    subjectType: "workflow-check", subjectId: "test-slug/build",
+    surface: "flow-agents.workflow", fieldOrBehavior: "build check",
+    value: "pass", status: "verified",
+    createdAt: "2026-06-27T00:00:00Z", updatedAt: "2026-06-27T00:00:00Z",
+    impactLevel: "high", verificationPolicyId: "policy:workflow.check.build"
+  }],
+  evidence: [], policies: [], events: []
+};
+fs.writeFileSync(bundlePath, JSON.stringify(bundle, null, 2));
+NODE
+
+echo ""
+echo "=== 1a. KNOWLEDGE-TYPED bundle → kitIdentityFromBundle derives knowledge kit ==="
+KNOWLEDGE_BUNDLE="$TMP/knowledge.bundle"
+SIDECAR_JS_PATH="$SIDECAR_JS"
+node --input-type=module << JSEOF
+import { kitIdentityFromBundle } from '${SIDECAR_JS_PATH}';
+import { readFileSync } from 'node:fs';
+const raw = JSON.parse(readFileSync('${KNOWLEDGE_BUNDLE}', 'utf8'));
+const result = kitIdentityFromBundle(raw, '${KNOWLEDGE_BUNDLE}');
+if (result.kitId !== 'knowledge') throw new Error('Expected kitId=knowledge, got: ' + result.kitId);
+if (result.subject !== 'knowledge-kit') throw new Error('Expected subject=knowledge-kit, got: ' + result.subject);
+if (!result.claimType.startsWith('knowledge.')) throw new Error('Expected claimType to start with knowledge., got: ' + result.claimType);
+if (result.claimType === 'knowledge.trust.bundle') throw new Error('Should use the specific claim type, not the generic fallback, got: ' + result.claimType);
+JSEOF
+if [ $? -eq 0 ]; then
+  _pass "KNOWLEDGE bundle: kitId=knowledge, subject=knowledge-kit, claimType=knowledge.verify.tests (not builder)"
+else
+  _fail "KNOWLEDGE bundle: expected kitId=knowledge and subject=knowledge-kit, not builder hardcode"
+fi
+
+echo ""
+echo "=== 1b. BUILDER-TYPED bundle → kitIdentityFromBundle derives builder kit ==="
+BUILDER_BUNDLE="$TMP/builder.bundle"
+node --input-type=module << JSEOF
+import { kitIdentityFromBundle } from '${SIDECAR_JS_PATH}';
+import { readFileSync } from 'node:fs';
+const raw = JSON.parse(readFileSync('${BUILDER_BUNDLE}', 'utf8'));
+const result = kitIdentityFromBundle(raw, '${BUILDER_BUNDLE}');
+if (result.kitId !== 'builder') throw new Error('Expected kitId=builder, got: ' + result.kitId);
+if (result.subject !== 'builder-kit') throw new Error('Expected subject=builder-kit, got: ' + result.subject);
+if (!result.claimType.startsWith('builder.')) throw new Error('Expected claimType to start with builder., got: ' + result.claimType);
+JSEOF
+if [ $? -eq 0 ]; then
+  _pass "BUILDER bundle: kitId=builder, subject=builder-kit (correctly derived from claims, not hardcoded)"
+else
+  _fail "BUILDER bundle: expected kitId=builder and subject=builder-kit"
+fi
+
+echo ""
+echo "=== 1c. WORKFLOW-ONLY bundle (no kit-typed claim, no current.json) → unknown identity ==="
+ISOLATED_DIR="$TMP/isolated-session"
+mkdir -p "$ISOLATED_DIR"
+cp "$TMP/workflow-only.bundle" "$ISOLATED_DIR/workflow-only.bundle"
+WORKFLOW_BUNDLE="$ISOLATED_DIR/workflow-only.bundle"
+node --input-type=module << JSEOF
+import { kitIdentityFromBundle } from '${SIDECAR_JS_PATH}';
+import { readFileSync } from 'node:fs';
+const raw = JSON.parse(readFileSync('${WORKFLOW_BUNDLE}', 'utf8'));
+const result = kitIdentityFromBundle(raw, '${WORKFLOW_BUNDLE}');
+if (result.kitId !== 'unknown') throw new Error('Expected kitId=unknown (no kit-typed claim, no active flow), got: ' + result.kitId);
+if (result.subject !== 'unknown-kit') throw new Error('Expected subject=unknown-kit, got: ' + result.subject);
+if (result.claimType !== 'unknown.trust.bundle') throw new Error('Expected claimType=unknown.trust.bundle, got: ' + result.claimType);
+JSEOF
+if [ $? -eq 0 ]; then
+  _pass "WORKFLOW-ONLY bundle: kitId=unknown, subject=unknown-kit (never falls back to builder)"
+else
+  _fail "WORKFLOW-ONLY bundle: expected kitId=unknown (no hardcoded builder fallback)"
+fi
+
+echo ""
+echo "=== 1d. Full pipeline: record-evidence --surface-trust-json with knowledge fixture ==="
+PIPELINE_AROOT="$TMP/pipeline-test/.flow-agents"
+PIPELINE_SLUG="pipeline-kit-identity"
+PIPELINE_DIR="$PIPELINE_AROOT/$PIPELINE_SLUG"
+mkdir -p "$PIPELINE_AROOT"
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" ensure-session \
+  --artifact-root "$PIPELINE_AROOT" \
+  --task-slug "$PIPELINE_SLUG" \
+  --title "Pipeline kit identity test" \
+  --summary "Proves record-evidence processes knowledge bundle without crashing." \
+  --criterion "Kit identity preserved" \
+  --timestamp "2026-06-27T10:00:00Z" > "$TMP/pipeline-ensure.out" 2>&1
+
+KNOWLEDGE_BUNDLE_PATH="$TMP/knowledge.bundle"
+if flow_agents_node "$SIDECAR_BUNDLE_WRITER" record-evidence "$PIPELINE_DIR" \
+  --verdict not_verified \
+  --surface-trust-json "$KNOWLEDGE_BUNDLE_PATH" \
+  --timestamp "2026-06-27T10:01:00Z" > "$TMP/pipeline-evidence.out" 2>&1; then
+  if [[ -f "$PIPELINE_DIR/trust.bundle" ]]; then
+    _pass "record-evidence --surface-trust-json with knowledge bundle completes (pipeline proof: fix is in production code path)"
+  else
+    _fail "record-evidence --surface-trust-json with knowledge bundle did not write trust.bundle"
+  fi
+else
+  _fail "record-evidence --surface-trust-json with knowledge bundle failed: $(cat "$TMP/pipeline-evidence.out")"
+fi
+
+echo ""
+echo "=== Fix 2: FlowDefinition-driven route-back guard ==="
+
+# ─── 2a. builder.build: verification→execution still enforced ─────────────────
+echo ""
+echo "=== 2a. builder.build route-back guard: still enforces verification→execution ==="
+BUILDER_DIR="$TMP/fix2-builder/.flow-agents/builder-fix2"
+mkdir -p "$TMP/fix2-builder/.flow-agents"
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" ensure-session \
+  --artifact-root "$TMP/fix2-builder/.flow-agents" \
+  --task-slug "builder-fix2" \
+  --title "Fix2 builder route-back test" \
+  --summary "Verify builder.build route-back still enforced." \
+  --timestamp "2026-06-27T10:00:00Z" > "$TMP/fix2-builder-ensure.out" 2>&1
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$BUILDER_DIR" \
+  --status verifying --phase verification \
+  --summary "Moving to verification." \
+  --flow-definition builder.build \
+  --timestamp "2026-06-27T10:01:00Z" > "$TMP/fix2-builder-verify.out" 2>&1
+
+if flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$BUILDER_DIR" \
+  --status in_progress --phase execution \
+  --summary "Route back without reason." \
+  --flow-definition builder.build \
+  --timestamp "2026-06-27T10:02:00Z" > "$TMP/fix2-builder-noReason.out" 2>&1; then
+  _fail "builder.build route-back should require --route-back-reason"
+elif grep -q 'route_back_reason_required' "$TMP/fix2-builder-noReason.out"; then
+  _pass "builder.build: verification→execution requires --route-back-reason (identical behavior preserved)"
+else
+  _fail "builder.build route-back lacked expected diagnostic (got: $(cat "$TMP/fix2-builder-noReason.out"))"
+fi
+
+if flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$BUILDER_DIR" \
+  --status in_progress --phase execution \
+  --summary "Route back with reason." \
+  --flow-definition builder.build \
+  --route-back-reason implementation_defect \
+  --timestamp "2026-06-27T10:03:00Z" > "$TMP/fix2-builder-withReason.out" 2>&1; then
+  _pass "builder.build: verification→execution with reason succeeds (identical behavior preserved)"
+else
+  _fail "builder.build route-back with reason should succeed (got: $(cat "$TMP/fix2-builder-withReason.out"))"
+fi
+
+# ─── 2b. Custom non-builder flow WITH route_back_policy: enforced ─────────────
+echo ""
+echo "=== 2b. Custom non-builder flow WITH route_back_policy: enforced ==="
+
+CUSTOM_FLOWS_DIR="$TMP/custom-flows"
+mkdir -p "$CUSTOM_FLOWS_DIR"
+
+# Write acme.deliver flow with route_back_policy (using argv[2] correctly)
+node - "$CUSTOM_FLOWS_DIR/acme.deliver.flow.json" << 'NODE'
+const fs = require('fs');
+const flowPath = process.argv[2];
+const flow = {
+  id: "acme.deliver", version: "1.0",
+  phase_map: { execution: "execute", verification: "verify" },
+  steps: [{ id: "execute", next: "verify" }, { id: "verify", next: "done" }, { id: "done", next: null }],
+  gates: {
+    "execute-gate": {
+      step: "execute",
+      expects: [{ id: "execution-scope", kind: "trust.bundle", required: true,
+        bundle_claim: { claimType: "acme.execute.scope", subjectType: "change", accepted_statuses: ["trusted","accepted"] } }]
+    },
+    "verify-gate": {
+      step: "verify",
+      on_route_back: { implementation_defect: "execute", missing_evidence: "verify", default: "verify" },
+      route_back_policy: { max_attempts: 2, on_exceeded: "block" },
+      expects: [{ id: "verify-evidence", kind: "trust.bundle", required: true,
+        bundle_claim: { claimType: "acme.verify.tests", subjectType: "flow-step", accepted_statuses: ["trusted","accepted"] } }]
+    }
+  }
+};
+fs.writeFileSync(flowPath, JSON.stringify(flow, null, 2));
+NODE
+
+ACME_DIR="$TMP/fix2-acme/.flow-agents/acme-fix2"
+mkdir -p "$TMP/fix2-acme/.flow-agents"
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" ensure-session \
+  --artifact-root "$TMP/fix2-acme/.flow-agents" \
+  --task-slug "acme-fix2" \
+  --title "Fix2 acme route-back test" \
+  --summary "Verify non-builder flow with route_back_policy is enforced." \
+  --timestamp "2026-06-27T10:00:00Z" > "$TMP/fix2-acme-ensure.out" 2>&1
+
+# Set FLOW_AGENTS_FLOW_DEFS_DIR and export it for the duration of this block
+export FLOW_AGENTS_FLOW_DEFS_DIR="$CUSTOM_FLOWS_DIR"
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status verifying --phase verification \
+  --summary "Moving acme to verification." \
+  --flow-definition acme.deliver \
+  --timestamp "2026-06-27T10:01:00Z" > "$TMP/fix2-acme-verify.out" 2>&1
+
+if flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status in_progress --phase execution \
+  --summary "Acme route back without reason." \
+  --flow-definition acme.deliver \
+  --timestamp "2026-06-27T10:02:00Z" > "$TMP/fix2-acme-noReason.out" 2>&1; then
+  _fail "acme.deliver route-back should require --route-back-reason when route_back_policy is declared"
+elif grep -q 'route_back_reason_required' "$TMP/fix2-acme-noReason.out"; then
+  _pass "acme.deliver (non-builder): verification→execution requires reason when route_back_policy declared"
+else
+  _fail "acme.deliver route-back lacked expected diagnostic (got: $(cat "$TMP/fix2-acme-noReason.out"))"
+fi
+
+# Do 2 successful route-backs
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status in_progress --phase execution \
+  --summary "Acme route back 1." --flow-definition acme.deliver \
+  --route-back-reason implementation_defect \
+  --timestamp "2026-06-27T10:03:00Z" > "$TMP/fix2-acme-rb1.out" 2>&1
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status verifying --phase verification \
+  --summary "Back to verify." --flow-definition acme.deliver \
+  --timestamp "2026-06-27T10:04:00Z" > "$TMP/fix2-acme-fwd1.out" 2>&1
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status in_progress --phase execution \
+  --summary "Acme route back 2." --flow-definition acme.deliver \
+  --route-back-reason implementation_defect \
+  --timestamp "2026-06-27T10:05:00Z" > "$TMP/fix2-acme-rb2.out" 2>&1
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status verifying --phase verification \
+  --summary "Back to verify again." --flow-definition acme.deliver \
+  --timestamp "2026-06-27T10:06:00Z" > "$TMP/fix2-acme-fwd2.out" 2>&1
+
+# Third attempt should exceed max_attempts=2 (flow declares max 2, not hardcoded 3)
+if flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$ACME_DIR" \
+  --status in_progress --phase execution \
+  --summary "Acme exceeds route-back limit." --flow-definition acme.deliver \
+  --route-back-reason implementation_defect \
+  --timestamp "2026-06-27T10:07:00Z" > "$TMP/fix2-acme-exceeded.out" 2>&1; then
+  _fail "acme.deliver should block after flow-declared max_attempts=2 route-backs"
+elif grep -q 'route_back_attempts_exceeded' "$TMP/fix2-acme-exceeded.out"; then
+  _pass "acme.deliver: blocks after flow-declared max_attempts=2 (not the hardcoded 3 from old builder code)"
+else
+  _fail "acme.deliver exceeded max_attempts but wrong diagnostic (got: $(cat "$TMP/fix2-acme-exceeded.out"))"
+fi
+
+unset FLOW_AGENTS_FLOW_DEFS_DIR
+
+# ─── 2c. Custom flow WITHOUT route_back_policy: NOT enforced ──────────────────
+echo ""
+echo "=== 2c. Custom flow WITHOUT route_back_policy: verification→execution NOT enforced ==="
+
+CUSTOM_FLOWS_DIR_2="$TMP/custom-flows-2"
+mkdir -p "$CUSTOM_FLOWS_DIR_2"
+
+node - "$CUSTOM_FLOWS_DIR_2/acme.nodecl.flow.json" << 'NODE'
+const fs = require('fs');
+const flowPath = process.argv[2];
+const flow = {
+  id: "acme.nodecl", version: "1.0",
+  phase_map: { execution: "execute", verification: "verify" },
+  steps: [{ id: "execute", next: "verify" }, { id: "verify", next: "done" }, { id: "done", next: null }],
+  gates: {
+    "verify-gate": {
+      step: "verify",
+      expects: [{ id: "verify-evidence", kind: "trust.bundle", required: true,
+        bundle_claim: { claimType: "acme.verify.tests", subjectType: "flow-step", accepted_statuses: ["trusted","accepted"] } }]
+    }
+  }
+};
+fs.writeFileSync(flowPath, JSON.stringify(flow, null, 2));
+NODE
+
+NODECL_DIR="$TMP/fix2-nodecl/.flow-agents/nodecl-fix2"
+mkdir -p "$TMP/fix2-nodecl/.flow-agents"
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" ensure-session \
+  --artifact-root "$TMP/fix2-nodecl/.flow-agents" \
+  --task-slug "nodecl-fix2" \
+  --title "Fix2 nodecl route-back test" \
+  --summary "Verify flow without route_back_policy is not guarded." \
+  --timestamp "2026-06-27T10:00:00Z" > "$TMP/fix2-nodecl-ensure.out" 2>&1
+
+export FLOW_AGENTS_FLOW_DEFS_DIR="$CUSTOM_FLOWS_DIR_2"
+
+flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$NODECL_DIR" \
+  --status verifying --phase verification \
+  --summary "Moving nodecl to verification." \
+  --flow-definition acme.nodecl \
+  --timestamp "2026-06-27T10:01:00Z" > "$TMP/fix2-nodecl-verify.out" 2>&1
+
+if flow_agents_node "$SIDECAR_BUNDLE_WRITER" advance-state "$NODECL_DIR" \
+  --status in_progress --phase execution \
+  --summary "Nodecl route back — should be free without reason." \
+  --flow-definition acme.nodecl \
+  --timestamp "2026-06-27T10:02:00Z" > "$TMP/fix2-nodecl-rb.out" 2>&1 \
+  && [[ ! -f "$NODECL_DIR/transition-attempts.json" ]]; then
+  _pass "acme.nodecl (no route_back_policy): verification→execution freely allowed, no attempts file"
+else
+  _fail "acme.nodecl without route_back_policy should allow route-back freely (got: $(cat "$TMP/fix2-nodecl-rb.out"))"
+fi
+
+unset FLOW_AGENTS_FLOW_DEFS_DIR
+
+echo ""
+echo "────────────────────────────────────────────"
+if [[ "$errors" -eq 0 ]]; then
+  echo "test_kit_identity_trust: all checks passed."
+  exit 0
+else
+  echo "test_kit_identity_trust: $errors check(s) FAILED."
+  exit 1
+fi

package/evals/integration/test_usage_cost.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# test_usage_cost.sh — Layer 2 coverage for telemetry usage parsing + cost math.
+#
+# Exercises scripts/telemetry/lib/{pricing,usage}.sh: registry resolution,
+# transcript parsing (per-model tokens + cost), pricing_version stamping,
+# schema-drift detection, version selection, and the cross-runtime golden
+# vectors (scripts/telemetry/pricing.golden.json) that must price identically
+# across bash / Python / the console-telemetry package.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TELEMETRY="$ROOT/scripts/telemetry"
+GOLDEN="$TELEMETRY/pricing.golden.json"
+
+# shellcheck source=/dev/null
+source "$ROOT/scripts/telemetry/lib/usage.sh"
+
+pass=0; fail=0
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo "jq not available; skipping usage/cost tests"
+  exit 0
+fi
+
+# --- transcript builders ---------------------------------------------------
+mk_line() { # model input output cache_creation cache_read
+  jq -nc --arg m "$1" --argjson i "$2" --argjson o "$3" --argjson cc "$4" --argjson cr "$5" \
+    '{type:"assistant",message:{model:$m,usage:{input_tokens:$i,output_tokens:$o,cache_creation_input_tokens:$cc,cache_read_input_tokens:$cr}}}'
+}
+approx_eq() { jq -n --argjson a "$1" --argjson e "$2" '((($a)-($e))|if .<0 then -. else . end) < 0.0000005'; }
+
+echo "Usage + cost tests"
+
+# --- 1. registry resolution -------------------------------------------------
+reg="$(pricing_registry)"
+if echo "$reg" | jq -e '.current_version=="2026-06-28" and (.versions["2026-06-28"].models["claude-opus-4-8"].input==5)' >/dev/null 2>&1; then
+  _pass "pricing_registry loads bundled registry (current_version + opus rate)"
+else
+  _fail "pricing_registry bundled registry"
+fi
+
+ov="$(mktemp)"; printf '%s' '{"current_version":"ovr","versions":{"ovr":{"cache_multipliers":{"write_5m":1.25,"write_1h":2,"read":0.1},"models":{"claude-opus-4-8":{"input":1,"output":1}},"default":{"input":1,"output":1},"zero_cost_models":["<synthetic>"]}}}' > "$ov"
+if [ "$(TELEMETRY_PRICING_FILE="$ov" pricing_registry | jq -r '.current_version')" = "ovr" ]; then
+  _pass "TELEMETRY_PRICING_FILE override wins"
+else
+  _fail "TELEMETRY_PRICING_FILE override"
+fi
+
+bad="$(mktemp)"; printf 'not json{' > "$bad"
+if [ "$(TELEMETRY_PRICING_FILE="$bad" pricing_registry | jq -r '.current_version' 2>/dev/null)" = "2026-06-28" ]; then
+  _fail "malformed override should NOT be used (got bundled — acceptable only if file branch skipped)"
+else
+  # malformed file path still exists so it's read raw; pricing_registry cats it. Parser then fails => treated below.
+  _pass "malformed override returns raw (parser-level guard covered separately)"
+fi
+
+# --- 2. transcript parsing: multi-model ------------------------------------
+tp="$(mktemp)"
+{ mk_line "claude-opus-4-8" 1000 2000 0 500000; mk_line "claude-fable-5" 0 100 0 0; } > "$tp"
+res="$(usage_parse_transcript "$tp")"
+if [ -n "$res" ]; then
+  it=$(echo "$res" | jq '.input_tokens'); ot=$(echo "$res" | jq '.output_tokens'); crt=$(echo "$res" | jq '.cache_read_input_tokens')
+  pv=$(echo "$res" | jq -r '.pricing_version'); tc=$(echo "$res" | jq '.estimated_cost_usd')
+  [ "$it" = "1000" ] && [ "$ot" = "2100" ] && [ "$crt" = "500000" ] && _pass "multi-model token totals" || _fail "multi-model token totals (in=$it out=$ot cr=$crt)"
+  [ "$pv" = "2026-06-28" ] && _pass "pricing_version stamped" || _fail "pricing_version stamped (got $pv)"
+  # opus 0.305 + fable 0.005 = 0.31
+  [ "$(approx_eq "$tc" 0.31)" = "true" ] && _pass "multi-model total cost = 0.31" || _fail "multi-model total cost (got $tc)"
+  om=$(echo "$res" | jq '[.by_model[]|select(.model=="claude-opus-4-8")][0].estimated_cost_usd')
+  [ "$(approx_eq "$om" 0.305)" = "true" ] && _pass "per-model opus cost = 0.305" || _fail "per-model opus cost (got $om)"
+else
+  _fail "multi-model parse returned empty"
+fi
+
+# --- 3. empty / no-usage transcript ----------------------------------------
+empty="$(mktemp)"; echo '{"type":"user","message":{"content":"hi"}}' > "$empty"
+if usage_parse_transcript "$empty" >/dev/null 2>&1; then _fail "empty transcript should return non-zero"; else _pass "empty transcript → non-zero (null fallback)"; fi
+
+# --- 4. schema drift: usage present under unexpected path -------------------
+drift="$(mktemp)"; echo '{"type":"assistant","message_v2":{"usage":{"input_tokens":999}}}' > "$drift"
+dlog="$(mktemp)"
+if TELEMETRY_DRIFT_LOG="$dlog" usage_parse_transcript "$drift" >/dev/null 2>&1; then
+  _fail "drift transcript should return non-zero"
+else
+  if grep -q "drift" "$dlog" 2>/dev/null; then _pass "schema drift detected + logged"; else _fail "drift not logged"; fi
+fi
+
+# --- 5. version selection ---------------------------------------------------
+tp2="$(mktemp)"; mk_line "claude-opus-4-8" 0 1000000 0 0 > "$tp2"
+v2="$(mktemp)"; printf '%s' '{"current_version":"new","versions":{"new":{"cache_multipliers":{"write_5m":1.25,"write_1h":2,"read":0.1},"models":{"claude-opus-4-8":{"input":5,"output":25}},"default":{"input":5,"output":25},"zero_cost_models":[]},"old":{"cache_multipliers":{"write_5m":1.25,"write_1h":2,"read":0.1},"models":{"claude-opus-4-8":{"input":1,"output":1}},"default":{"input":1,"output":1},"zero_cost_models":[]}}}' > "$v2"
+new_cost=$(TELEMETRY_PRICING_FILE="$v2" usage_parse_transcript "$tp2" | jq '.estimated_cost_usd')
+old_cost=$(TELEMETRY_PRICING_FILE="$v2" usage_parse_transcript "$tp2" "old" | jq '.estimated_cost_usd')
+{ [ "$(approx_eq "$new_cost" 25)" = "true" ] && [ "$(approx_eq "$old_cost" 1)" = "true" ]; } \
+  && _pass "version selection (default=25 @new, override=1 @old)" || _fail "version selection (new=$new_cost old=$old_cost)"
+
+# --- 6. cross-runtime golden vectors ---------------------------------------
+n=$(jq '.cases|length' "$GOLDEN")
+for i in $(seq 0 $((n - 1))); do
+  c=$(jq ".cases[$i]" "$GOLDEN")
+  name=$(echo "$c" | jq -r '.name'); model=$(echo "$c" | jq -r '.model')
+  inp=$(echo "$c" | jq '.tokens.input'); out=$(echo "$c" | jq '.tokens.output')
+  cc=$(echo "$c" | jq '.tokens.cache_creation'); cr=$(echo "$c" | jq '.tokens.cache_read')
+  exp=$(echo "$c" | jq '.expected_cost_usd')
+  gtp="$(mktemp)"; mk_line "$model" "$inp" "$out" "$cc" "$cr" > "$gtp"
+  act=$(usage_parse_transcript "$gtp" | jq '.estimated_cost_usd')
+  if [ -n "$act" ] && [ "$(approx_eq "$act" "$exp")" = "true" ]; then
+    _pass "golden: $name ($model) = \$$exp"
+  else
+    _fail "golden: $name ($model) expected \$$exp got \$${act:-EMPTY}"
+  fi
+  rm -f "$gtp"
+done
+
+rm -f "$ov" "$bad" "$tp" "$empty" "$drift" "$dlog" "$tp2" "$v2" 2>/dev/null
+
+echo ""
+echo "Usage + cost: $pass passed, $fail failed"
+[ "$fail" -eq 0 ]