@kontourai/flow-agents 3.12.1 → 4.1.0

package/evals/integration/test_narrative_evals.sh

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+# Grounded narrative faithfulness eval suite: one-command frozen corpus replay,
+# schema-valid result emission, and the R3 scorer-teeth mutation battery (#612).
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+FIXTURES="$ROOT/evals/fixtures/narrative-evals"
+SCORER="$FIXTURES/scorer.mjs"
+TMP="$(mktemp -d)"
+BACKUP="$TMP/scorer.mjs.original"
+errors=0
+
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+restore_scorer() { if [[ -f "$BACKUP" ]]; then cp "$BACKUP" "$SCORER"; fi; }
+cleanup() { restore_scorer; rm -rf "$TMP"; }
+trap cleanup EXIT
+
+echo "Narrative grounded evals integration"
+if npm run build --silent; then _pass "TypeScript build completed"; else _fail "TypeScript build failed"; fi
+cp "$SCORER" "$BACKUP"
+
+# AC1: one command replays the whole frozen corpus and emits a schema-valid result.
+if NARRATIVE_EVAL_RESULT_OUT="$TMP/result.json" node "$SCORER" all >"$TMP/scorer.out" 2>"$TMP/scorer.err"; then
+  _pass "AC1/AC2: frozen corpus meets every answer-key verdict"
+else
+  _fail "corpus replay failed: $(cat "$TMP/scorer.out" "$TMP/scorer.err")"
+fi
+
+if grep -q 'fixture provenance: kontourai/flow-agents#612 narrative-evals-corpus/v1' "$TMP/scorer.out" \
+  && grep -q 'scorer counts: accept=9 reject=5 known_gap=1 total=15' "$TMP/scorer.out" \
+  && grep -q 'unsupported observed-claim rate=0' "$TMP/scorer.out" \
+  && grep -q 'citation resolvability=1' "$TMP/scorer.out" \
+  && grep -q 'material-claim coverage=1' "$TMP/scorer.out" \
+  && grep -q 'epistemic classification accuracy=1' "$TMP/scorer.out"; then
+  _pass "R7/AC5: deterministic provenance, counts, and threshold lines present"
+else
+  _fail "deterministic report is incomplete: $(cat "$TMP/scorer.out")"
+fi
+
+# D8: the contradictory case is disclosed as a known_gap, never faked as detection.
+if grep -q 'contradictory: expected=known_gap actual=known_gap' "$TMP/scorer.out"; then
+  _pass "D8: contradictory fixture recorded as a documented known_gap"
+else
+  _fail "D8 contradictory fixture was not scored as a known_gap"
+fi
+
+# AC3/R7: capability parity is DECLARED (queryCapability), not behavior-probed, for >=2 runtimes.
+if grep -q 'capability parity: claude-code/intent_annotation=unsupported' "$TMP/scorer.out" \
+  && grep -q 'capability parity: claude-code/per_delegation_trace_context=unsupported' "$TMP/scorer.out" \
+  && grep -q 'capability parity: codex/intent_annotation=unsupported' "$TMP/scorer.out" \
+  && grep -q 'capability parity: codex/per_delegation_trace_context=unsupported' "$TMP/scorer.out"; then
+  _pass "AC3/R7: declared capability parity emitted for claude-code and codex"
+else
+  _fail "AC3 capability parity block is incomplete: $(cat "$TMP/scorer.out")"
+fi
+
+# R2/R6/AC6: the emitted result validates against the shipped schema in CI too.
+if [[ -f "$TMP/result.json" ]] && node --input-type=module - "$ROOT" "$TMP/result.json" >"$TMP/schema.out" 2>&1 <<'NODE'
+import { pathToFileURL } from 'node:url';
+import path from 'node:path';
+import fs from 'node:fs';
+const [root, file] = process.argv.slice(2);
+const api = await import(pathToFileURL(path.join(root, 'build/src/index.js')));
+const value = JSON.parse(fs.readFileSync(file, 'utf8'));
+if (value.schema_version !== 'narrative-eval-result/v1') { console.error('unexpected schema_version'); process.exit(1); }
+const issues = api.validateNarrativeEvalResult(value);
+if (issues.length > 0) { console.error(JSON.stringify(issues)); process.exit(1); }
+// Negative control: a broken result must be rejected, proving the validator has teeth.
+const broken = structuredClone(value); broken.metrics.unsupported_claim_rate = 2; broken.injected = true;
+if (api.validateNarrativeEvalResult(broken).length === 0) { console.error('schema validator accepted a malformed result'); process.exit(1); }
+NODE
+then
+  _pass "R2/R6/AC6: emitted result validates against narrative-eval-result/v1 in-process and in CI"
+else
+  _fail "emitted result failed schema validation: $(cat "$TMP/schema.out" 2>/dev/null || echo 'result.json missing')"
+fi
+
+# R2 determinism: the emitted artifact must be byte-identical across runs (content-
+# addressed fa1 ids must not leak the random temp path). Guards review finding HIGH-2.
+NARRATIVE_EVAL_RESULT_OUT="$TMP/result-a.json" node "$SCORER" all >/dev/null 2>&1 || true
+NARRATIVE_EVAL_RESULT_OUT="$TMP/result-b.json" node "$SCORER" all >/dev/null 2>&1 || true
+if cmp -s "$TMP/result-a.json" "$TMP/result-b.json"; then
+  _pass "R2: emitted result is byte-identical across runs (deterministic artifact)"
+else
+  _fail "emitted result is non-deterministic across runs: $(diff "$TMP/result-a.json" "$TMP/result-b.json" | head -5)"
+fi
+
+# HIGH-1 guard: material_claim_coverage must be COMPUTED from the canonical extraction,
+# not a lockstep counter — removing a covering statement must drop covered below total.
+if node --input-type=module - "$ROOT" >"$TMP/covteeth.out" 2>&1 <<'NODE'
+import { pathToFileURL } from 'node:url';
+import path from 'node:path';
+import fs from 'node:fs';
+import os from 'node:os';
+import { createHash } from 'node:crypto';
+const [root] = process.argv.slice(2);
+const api = await import(pathToFileURL(path.join(root, 'build/src/index.js')));
+const sha8 = (b) => createHash('sha256').update(b).digest('hex').slice(0, 8);
+const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'cov-teeth-'));
+const sessionDir = path.join(temp, 'session'); const narrativeDir = path.join(temp, 'narrative');
+fs.mkdirSync(sessionDir, { recursive: true });
+const line = JSON.stringify({ command: 'npm test', result: 'fail', exitCode: 1 });
+fs.writeFileSync(path.join(sessionDir, 'command-log.jsonl'), `${line}\n`);
+const ref = `fa1:cmdlog:session:line-1/${sha8(Buffer.from(line))}`;
+const at = '2026-01-01T00:00:00.000Z';
+api.snapshotNarrative({ narrativeDir, narrativeId: 'cov', requests: [{ source: api.parseSourceId(ref), roots: { sessionDir } }], redactionFields: [], compiler: { name: 'cov', version: '1', policy_hash: 'fixture' }, captureCompleteness: { channels: { full: 'active' }, known_gaps: [] } }, { now: () => at });
+const full = api.composeGroundedNarrative(narrativeDir, { compiledAt: at });
+const covFull = api.materialEventCoverage(full, narrativeDir);
+const stripped = JSON.parse(JSON.stringify(full));
+const rt = stripped.sections.find((s) => s.authority === 'flow-agents' && s.kind === 'runtime-projection');
+rt.embedded.turns = rt.embedded.turns.map((t) => ({ ...t, statements: t.statements.filter((s) => !/was observed to fail/.test(s.proposition)) }));
+rt.embedded.document_statements = rt.embedded.document_statements.filter((s) => !/was observed to fail/.test(s.proposition));
+const covStripped = api.materialEventCoverage(stripped, narrativeDir);
+fs.rmSync(temp, { recursive: true, force: true });
+if (!(covFull.total >= 1 && covFull.covered === covFull.total && covStripped.total === covFull.total && covStripped.covered < covStripped.total)) {
+  console.error(`coverage metric has no teeth: full=${covFull.covered}/${covFull.total} stripped=${covStripped.covered}/${covStripped.total}`); process.exit(1);
+}
+NODE
+then
+  _pass "HIGH-1: material_claim_coverage is computed from the canonical extraction (drops when coverage is removed)"
+else
+  _fail "material_claim_coverage does not react to removed coverage: $(cat "$TMP/covteeth.out")"
+fi
+
+# R3/AC2: the mutation battery. For EACH corruption class, disable the scorer's
+# detection at its named /* eval-check:<name> */ anchor, confirm the compiled
+# scorer still parses, confirm the matching reject fixture flips to accept
+# (teeth), then restore byte-for-byte and confirm the corpus rejects it again.
+mutation_test() {
+  local check="$1" anchor="$2" call="$3" fixture="$4"
+  restore_scorer
+  node - "$SCORER" "$anchor" "$call" <<'NODE'
+const fs = require('fs');
+const [file, anchor, call] = process.argv.slice(2);
+let text = fs.readFileSync(file, 'utf8');
+const needle = `${anchor}\n  ${call}`;
+if (!text.includes(needle)) throw new Error(`mutation target missing: ${needle}`);
+text = text.replace(needle, `${anchor}\n  // mutation: check deliberately disabled`);
+fs.writeFileSync(file, text);
+NODE
+  if ! node --check "$SCORER" >/dev/null; then
+    _fail "${check} mutation produced invalid JavaScript"
+  elif node "$SCORER" "$check" >"$TMP/mutation-$check.out" 2>"$TMP/mutation-$check.err"; then
+    _fail "${check} scorer still passed after its detection was disabled"
+  elif grep -q "${fixture}: expected=reject actual=accept" "$TMP/mutation-$check.out"; then
+    _pass "scorer teeth: disabling ${check} flips ${fixture} reject to accept"
+  else
+    _fail "${check} mutation failed for an unrelated reason: $(cat "$TMP/mutation-$check.out" "$TMP/mutation-$check.err")"
+  fi
+  restore_scorer
+  if cmp -s "$BACKUP" "$SCORER" && node "$SCORER" "$check" >/dev/null 2>&1; then
+    _pass "${check} detection restored byte-for-byte and the corpus rejects again"
+  else
+    _fail "${check} detection did not restore cleanly"
+  fi
+}
+
+mutation_test support   '/* eval-check:support */'   'supportCheck(state, statements);'   hallucinated-statement
+mutation_test citation  '/* eval-check:citation */'  'citationCheck(state, verdict);'     dangling-citation
+mutation_test coverage  '/* eval-check:coverage */'  'coverageCheck(state, verdict);'     omitted-failure
+mutation_test epistemic '/* eval-check:epistemic */' 'epistemicCheck(state, verdict);'    mislabeled-inference
+mutation_test injection '/* eval-check:injection */' 'injectionCheck(state, injection);'  injection-followed
+
+restore_scorer
+cmp -s "$BACKUP" "$SCORER" || _fail "scorer was not restored after the mutation battery"
+
+echo ""
+if [[ "$errors" -eq 0 ]]; then
+  echo "narrative grounded evals tests passed: 15/15."
+  exit 0
+fi
+echo "narrative grounded evals tests FAILED: $errors issue(s)."
+exit 1

package/evals/integration/test_narrative_grounded_envelope.sh

@@ -0,0 +1,280 @@
+#!/usr/bin/env bash
+# Frozen authorities compose into a deterministic, grounded execution narrative.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+FIXTURES="$ROOT/evals/fixtures/narrative-sources"
+TMP="$(mktemp -d)"
+errors=0
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+trap 'rm -rf "$TMP"' EXIT
+
+sha256_file() { node -e "const fs=require('fs'),c=require('crypto');process.stdout.write(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'))" "$1"; }
+sha8_line() { node -e "const fs=require('fs'),c=require('crypto');const l=fs.readFileSync(process.argv[1],'utf8').trimEnd().split(/\\r?\\n/)[Number(process.argv[2])];process.stdout.write(c.createHash('sha256').update(l).digest('hex').slice(0,8))" "$1" "$2"; }
+telemetry_pin() { node -e "const fs=require('fs'),c=require('crypto');const l=fs.readFileSync(process.argv[1],'utf8').split('\\n').find(x=>x.trim()&&JSON.parse(x).event_id===process.argv[2]);process.stdout.write(c.createHash('sha256').update(l).digest('hex').slice(0,8))" "$1" "$2"; }
+json_assert() {
+  node - "$1" "$2" <<'NODE'
+const fs = require('fs');
+const value = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
+if (!Function('value', `return Boolean(${process.argv[3]})`)(value)) process.exit(1);
+NODE
+}
+
+echo "Narrative grounded envelope integration"
+if npm run build --silent; then _pass "TypeScript build completed"; else _fail "TypeScript build failed"; fi
+
+RAW="$TMP/raw"
+ARTIFACT_ROOT="$TMP/artifacts"
+SESSION="$RAW/fixture-session"
+AGENTS_DIR="$SESSION/agents"
+NESTED_EVENTS="$AGENTS_DIR/nested-worker/events.jsonl"
+TELEMETRY="$RAW/telemetry"
+FLOW="$RAW/flow"
+REPO="$RAW/repo"
+TRANSCRIPT="$RAW/canary.txt"
+mkdir -p "$RAW"
+cp -R "$FIXTURES/session" "$SESSION"
+cp -R "$FIXTURES/telemetry" "$TELEMETRY"
+cp -R "$FIXTURES/flow" "$FLOW"
+cp -R "$FIXTURES/repo" "$REPO"
+printf '%s\n' 'AC6_CANARY' >"$TRANSCRIPT"
+
+# A foreign authority blob containing a configured sensitive key must be
+# unavailable as a whole. Its verbatim bytes must never reach the snapshot.
+FOREIGN_REPORT_FILE="$FLOW/runs/run-redacted/report.json"
+mkdir -p "$(dirname "$FOREIGN_REPORT_FILE")"
+printf '%s\n' '{"run_id":"run-redacted","nested":{"foreignSecret":"AC6_CANARY"}}' >"$FOREIGN_REPORT_FILE"
+
+# Add one attributed nested-worker event beside the fixture's lineage-less event.
+printf '%s\n' '{"kind":"delegation","at":"2026-07-14T13:00:03.000Z","target":"attributed-leaf","task":"attributed fixture","lineage":{"actor":"nested-worker"}}' >>"$NESTED_EVENTS"
+
+# Genuine chained observations preserve fail -> retry.
+node - "$ROOT/scripts/lib/command-log-chain.js" "$SESSION/command-log.jsonl" <<'NODE'
+const fs = require('fs');
+const chain = require(process.argv[2]);
+const records = [
+  { command: 'npm test', result: 'fail', exitCode: 1, capturedAt: '2026-07-14T13:00:04.000Z', source: 'narrative-envelope-fixture' },
+  { command: 'npm test', result: 'pass', exitCode: 0, capturedAt: '2026-07-14T13:00:05.000Z', source: 'narrative-envelope-fixture' },
+];
+let previous = chain.CHAIN_GENESIS;
+for (let index = 0; index < records.length; index += 1) {
+  const record = records[index];
+  const hash = chain.computeChainHash(previous, record);
+  record._chain = { seq: index, prevHash: previous, hash };
+  previous = hash;
+}
+fs.writeFileSync(process.argv[3], records.map(JSON.stringify).join('\n') + '\n');
+NODE
+
+CMD0="$(node -e "const r=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8').split('\\n')[0]);process.stdout.write(r._chain.hash.slice(0,8))" "$SESSION/command-log.jsonl")"
+CMD1="$(node -e "const r=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8').split('\\n')[1]);process.stdout.write(r._chain.hash.slice(0,8))" "$SESSION/command-log.jsonl")"
+NESTED0="$(sha8_line "$NESTED_EVENTS" 0)"
+NESTED1="$(sha8_line "$NESTED_EVENTS" 1)"
+FLOW_STATE="$(sha256_file "$FLOW/runs/run-runtime/state.json")"; FLOW_STATE="${FLOW_STATE:0:8}"
+FLOW_REPORT="$(sha256_file "$FLOW/runs/run-fixture/report.json")"; FLOW_REPORT="${FLOW_REPORT:0:8}"
+FOREIGN_REPORT="$(sha256_file "$FOREIGN_REPORT_FILE")"; FOREIGN_REPORT="${FOREIGN_REPORT:0:8}"
+BUNDLE="$(sha256_file "$SESSION/trust.bundle")"; BUNDLE="${BUNDLE:0:8}"
+FILE_SHA="$(sha256_file "$REPO/created.txt")"
+MISSING_SHA="$(printf 'missing fixture' | node -e "const c=require('crypto');let d='';process.stdin.on('data',x=>d+=x).on('end',()=>process.stdout.write(c.createHash('sha256').update(d).digest('hex')))")"
+PATH_HASH="$(node -e "const c=require('crypto'),p=require('path');process.stdout.write(c.createHash('sha256').update(p.resolve(process.argv[1])).digest('hex').slice(0,8))" "$TRANSCRIPT")"
+TURN="$(telemetry_pin "$TELEMETRY/runtime.jsonl" runtime-turn-explicit)"
+TIMEOUT="$(telemetry_pin "$TELEMETRY/runtime.jsonl" runtime-tool-timeout)"
+DERIVED="$(telemetry_pin "$TELEMETRY/runtime.jsonl" runtime-turn-derived)"
+END="$(telemetry_pin "$TELEMETRY/runtime.jsonl" runtime-session-end)"
+MCP="$(telemetry_pin "$TELEMETRY/full.jsonl" evt-mcp-gap)"
+
+SOURCES=(
+  "fa1:telemetry:runtime/runtime-session:runtime-turn-explicit/$TURN"
+  "fa1:telemetry:runtime/runtime-session:runtime-tool-timeout/$TIMEOUT"
+  "fa1:flow-state:run-runtime:state/$FLOW_STATE"
+  "fa1:flow-report:run-fixture:report/$FLOW_REPORT"
+  "fa1:flow-report:run-redacted:report/$FOREIGN_REPORT"
+  "fa1:surface-explanation:fixture-session/$BUNDLE:claim-fixture"
+  "fa1:delegation:fixture-session/nested-worker:0/$NESTED0"
+  "fa1:delegation:fixture-session/nested-worker:1/$NESTED1"
+  "fa1:telemetry:runtime/runtime-session:runtime-turn-derived/$DERIVED"
+  "fa1:telemetry:runtime/runtime-session:runtime-session-end/$END"
+  "fa1:cmdlog:fixture-session:0/$CMD0"
+  "fa1:cmdlog:fixture-session:1/$CMD1"
+  "fa1:file:created.txt:$FILE_SHA"
+  "fa1:file:missing.txt:$MISSING_SHA"
+  "fa1:telemetry:full/session-fixture:evt-mcp-gap/$MCP"
+  "fa1:transcript:$PATH_HASH:0-$(wc -c <"$TRANSCRIPT" | tr -d ' ')"
+)
+ARGS=(narrative-sources snapshot --artifact-root "$ARTIFACT_ROOT" --narrative-id grounded-fixture
+  --telemetry-root "$TELEMETRY" --session-root "$SESSION" --flow-root "$FLOW" --repo-root "$REPO"
+  --transcript-path "$TRANSCRIPT" --redact-fields foreignSecret
+  --capture-completeness "$FIXTURES/expected-capture-completeness.json")
+for source in "${SOURCES[@]}"; do ARGS+=(--source "$source"); done
+
+export https_proxy=http://127.0.0.1:9 http_proxy=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 HTTP_PROXY=http://127.0.0.1:9 NO_PROXY='*'
+if node "$ROOT/build/src/cli.js" "${ARGS[@]}" >"$TMP/snapshot.json" 2>"$TMP/snapshot.err"; then _pass "AC9: snapshot succeeds with network disabled"; else _fail "snapshot failed: $(<"$TMP/snapshot.err")"; fi
+
+NARRATIVE_DIR="$ARTIFACT_ROOT/.kontourai/narrative/grounded-fixture"
+OUT1="$TMP/out-one"; OUT2="$TMP/out-two"
+COMPILED_AT="2026-07-14T16:00:00.000Z"
+if node "$ROOT/build/src/cli.js" narrative-sources compose --narrative-dir "$NARRATIVE_DIR" --compiled-at "$COMPILED_AT" --out-dir "$OUT1" --render >"$TMP/compose-one.json" 2>"$TMP/compose-one.err"; then _pass "AC1/AC9: compose validates and renders offline"; else _fail "compose failed: $(<"$TMP/compose-one.err")"; fi
+if node "$ROOT/build/src/cli.js" narrative-sources compose --narrative-dir "$NARRATIVE_DIR" --compiled-at "$COMPILED_AT" --out-dir "$OUT2" --render >"$TMP/compose-two.json" 2>"$TMP/compose-two.err"; then _pass "AC7: second pinned compile succeeds"; else _fail "second compose failed: $(<"$TMP/compose-two.err")"; fi
+ENVELOPE1="$(node -e "process.stdout.write(JSON.parse(require('fs').readFileSync(process.argv[1])).envelopePath)" "$TMP/compose-one.json")"
+RENDER1="$(node -e "process.stdout.write(JSON.parse(require('fs').readFileSync(process.argv[1])).renderPath)" "$TMP/compose-one.json")"
+ENVELOPE2="$(node -e "process.stdout.write(JSON.parse(require('fs').readFileSync(process.argv[1])).envelopePath)" "$TMP/compose-two.json")"
+
+if cmp -s "$ENVELOPE1" "$ENVELOPE2"; then _pass "AC7: identical compiled-at produces byte-identical envelopes"; else _fail "AC7: deterministic envelopes differ"; fi
+if json_assert "$ENVELOPE1" 'value.schema_version === "grounded-execution-narrative/v1" && ["manifest_sha256","schema_sha256","config_sha256","compiler_sha256"].every(k => /^[0-9a-f]{64}$/.test(value.provenance[k]))'; then _pass "AC1/AC7: schema and provenance hashes are present"; else _fail "AC1/AC7: envelope provenance is incomplete"; fi
+if json_assert "$ENVELOPE1" 'JSON.stringify(value.sections.find(s=>s.authority==="flow-agents").embedded).includes("was observed to fail") && JSON.stringify(value).includes("was retried across 2 attempts") && JSON.stringify(value).includes("30000 ms timeout") && JSON.stringify(value).includes("classified as a no-op") && JSON.stringify(value).includes("created.txt")'; then _pass "AC3: failure, retry, timeout, no-op, and created-file facts survive"; else _fail "AC3: material runtime fact was dropped"; fi
+if json_assert "$ENVELOPE1" 'JSON.stringify(value).includes("nested-worker") && [...value.sections.find(s=>s.authority==="flow-agents").embedded.turns.flatMap(t=>t.statements),...value.sections.find(s=>s.authority==="flow-agents").embedded.document_statements].some(s=>s.actor==="unattributed")'; then _pass "AC5: attributed and unattributed multi-agent facts survive"; else _fail "AC5: agent attribution disclosure is incomplete"; fi
+if ! grep -R -q 'AC6_CANARY' "$NARRATIVE_DIR" "$OUT1" "$OUT2" \
+  && grep -q 'redacted' "$RENDER1" \
+  && node - "$NARRATIVE_DIR/source-manifest.json" "${SOURCES[4]}" <<'NODE'
+const fs=require('fs'); const manifest=JSON.parse(fs.readFileSync(process.argv[2]));
+const entry=manifest.sources.find(source=>source.source_id===process.argv[3]);
+if (!entry || entry.status!=='unavailable' || entry.unavailable_reason!=='redacted' || entry.sha256!==undefined) process.exit(1);
+NODE
+then _pass "AC6: foreign authority canary is wholly unavailable/redacted and absent"; else _fail "AC6: foreign authority canary leaked or was partially captured"; fi
+
+# Every reference either resolves hash-verified or is the manifest-declared unavailable source.
+if node --input-type=module - "$NARRATIVE_DIR" "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs';
+const dir=process.argv[2], env=JSON.parse(fs.readFileSync(process.argv[3])), api=await import(`file://${process.argv[4]}`);
+const manifest=JSON.parse(fs.readFileSync(`${dir}/source-manifest.json`));
+const unavailable=new Set(manifest.sources.filter(s=>s.status==='unavailable').map(s=>s.source_id));
+const refs=new Set(JSON.stringify(env).match(/fa1:[A-Za-z0-9%._~:/#-]+/g)??[]);
+for (const ref of refs) { const r=api.resolveSource(dir,ref); if (r.status!=='resolved' && !unavailable.has(ref)) process.exit(1); }
+NODE
+then _pass "AC1: factual references resolve or carry declared unavailability"; else _fail "AC1: unresolved factual reference"; fi
+
+# Removing grounding must fail the public validator.
+if node --input-type=module - "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs'; const env=JSON.parse(fs.readFileSync(process.argv[2])); delete env.conclusions[0].grounding;
+const api=await import(`file://${process.argv[3]}`); process.exit(api.validateGroundedNarrative(env).length ? 0 : 1);
+NODE
+then _pass "AC4: grounding-less conclusion is rejected"; else _fail "AC4: grounding-less conclusion validated"; fi
+
+# A syntactically valid but nonexistent authority ref must not ground a
+# conclusion, even when another section of that authority exists.
+if node --input-type=module - "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs'; const env=JSON.parse(fs.readFileSync(process.argv[2]));
+env.conclusions[0].grounding={kind:'flow_gate_derivation',source_ref:'fa1:flow-report:missing:report/00000000',pointer:'/gates/0'};
+const api=await import(`file://${process.argv[3]}`); process.exit(api.validateGroundedNarrative(env).length ? 0 : 1);
+NODE
+then _pass "AC4: nonexistent grounding source_ref is rejected"; else _fail "AC4: nonexistent grounding source_ref validated"; fi
+
+# A grounding pointer must RFC6901-resolve within the exact frozen Flow blob.
+if node --input-type=module - "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs'; const env=JSON.parse(fs.readFileSync(process.argv[2]));
+const conclusion=env.conclusions.find(item=>item.grounding?.kind==='flow_gate_derivation');
+if (!conclusion) process.exit(1);
+conclusion.grounding.pointer='/definitely/not/present';
+const api=await import(`file://${process.argv[3]}`); process.exit(api.validateGroundedNarrative(env).length ? 0 : 1);
+NODE
+then _pass "AC4: nonexistent RFC6901 grounding pointer is rejected"; else _fail "AC4: nonexistent grounding pointer validated"; fi
+
+# An existing Flow pointer is still invalid grounding unless its resolved value
+# has the typed gate-summary shape consumed by the derivation.
+if node --input-type=module - "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs'; const env=JSON.parse(fs.readFileSync(process.argv[2]));
+const conclusion=env.conclusions.find(item=>item.grounding?.kind==='flow_gate_derivation');
+if (!conclusion) process.exit(1);
+conclusion.grounding.pointer='/run_id';
+const api=await import(`file://${process.argv[3]}`); process.exit(api.validateGroundedNarrative(env).length ? 0 : 1);
+NODE
+then _pass "AC4: existing non-gate Flow pointer is rejected"; else _fail "AC4: non-gate Flow pointer validated"; fi
+
+# A proposition is a deterministic derivation of its Flow gate fields, not
+# caller-supplied prose, even when its source and pointer otherwise validate.
+if node --input-type=module - "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs'; const env=JSON.parse(fs.readFileSync(process.argv[2]));
+const conclusion=env.conclusions.find(item=>item.grounding?.kind==='flow_gate_derivation');
+if (!conclusion) process.exit(1);
+conclusion.proposition='Gate result was replaced with free prose.';
+const api=await import(`file://${process.argv[3]}`); process.exit(api.validateGroundedNarrative(env).length ? 0 : 1);
+NODE
+then _pass "AC4: mismatched Flow proposition is rejected"; else _fail "AC4: mismatched Flow proposition validated"; fi
+
+# Surface conclusions carry the same deterministic proposition discipline.
+if node --input-type=module - "$ENVELOPE1" "$ROOT/build/src/index.js" <<'NODE'
+import fs from 'node:fs'; const env=JSON.parse(fs.readFileSync(process.argv[2]));
+const conclusion=env.conclusions.find(item=>item.grounding?.kind==='surface_explanation');
+if (!conclusion) process.exit(1);
+conclusion.proposition='Claim result was replaced with free prose.';
+const api=await import(`file://${process.argv[3]}`); process.exit(api.validateGroundedNarrative(env).length ? 0 : 1);
+NODE
+then _pass "AC4: mismatched Surface proposition is rejected"; else _fail "AC4: mismatched Surface proposition validated"; fi
+
+# A timezone-invalid window must retain the telemetry refs that made it
+# unusable so the no_timezone derivation has complete, inspectable inputs.
+if node --input-type=module - "$TMP/timezone-correlation" "$ROOT/build/src/index.js" <<'NODE'
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+const root=process.argv[2], api=await import(`file://${process.argv[3]}`);
+const narrativeDir=path.join(root,'narrative'), telemetryDir=path.join(root,'telemetry'), flowRoot=path.join(root,'flow');
+fs.mkdirSync(telemetryDir,{recursive:true}); fs.mkdirSync(path.join(flowRoot,'runs','timezone-run'),{recursive:true});
+const telemetry=[
+  {session_id:'timezone-session',event_id:'turn-start',event_type:'turn.user',timestamp:'2026-07-14T13:00:00.000',hook:{turn_id:'turn-one'}},
+  {session_id:'timezone-session',event_id:'turn-end',event_type:'tool.result',timestamp:'2026-07-14T13:00:20.000',hook:{turn_id:'turn-one'},tool:{name:'read'}},
+];
+const lines=telemetry.map(JSON.stringify);
+fs.writeFileSync(path.join(telemetryDir,'full.jsonl'),`${lines.join('\n')}\n`);
+const stateBytes=Buffer.from(JSON.stringify({run_id:'timezone-run',session_id:'timezone-session',transitions:[{from:'planned',to:'executing',at:'2026-07-14T13:00:10.000Z'}]}));
+const reportBytes=Buffer.from(JSON.stringify({run_id:'timezone-run',gate_summaries:[]}));
+fs.writeFileSync(path.join(flowRoot,'runs','timezone-run','state.json'),stateBytes);
+fs.writeFileSync(path.join(flowRoot,'runs','timezone-run','report.json'),reportBytes);
+const sha8=bytes=>createHash('sha256').update(bytes).digest('hex').slice(0,8);
+const telemetryRefs=telemetry.map((event,index)=>`fa1:telemetry:full/timezone-session:${event.event_id}/${sha8(Buffer.from(lines[index]))}`);
+const requests=telemetryRefs.map(sourceId=>({source:api.parseSourceId(sourceId),roots:{telemetryDir}}));
+requests.push(
+  {source:api.parseSourceId(`fa1:flow-state:timezone-run:state/${sha8(stateBytes)}`),roots:{flowRoot}},
+  {source:api.parseSourceId(`fa1:flow-report:timezone-run:report/${sha8(reportBytes)}`),roots:{flowRoot}},
+);
+api.snapshotNarrative({
+  narrativeDir,narrativeId:'timezone-correlation',requests,redactionFields:[],
+  compiler:{name:'grounded-envelope-integration',version:'1',policy_hash:'fixture'},
+  captureCompleteness:{channels:{full:'active'},known_gaps:[]},
+},{now:()=> '2026-07-14T15:00:00.000Z'});
+const envelope=api.composeGroundedNarrative(narrativeDir,{compiledAt:'2026-07-14T16:00:00.000Z'});
+const unplaced=envelope.correlation.unplaced.filter(item=>item.reason==='no_timezone');
+if (unplaced.length!==1) process.exit(1);
+if (!telemetryRefs.every(ref=>unplaced.every(item=>item.rule.inputs.includes(ref)))) process.exit(1);
+NODE
+then _pass "AC correlation: no_timezone rule inputs retain offending telemetry refs"; else _fail "AC correlation: no_timezone rule inputs omit offending telemetry refs"; fi
+
+# Extract the JSON string without normalization, then compare its decoded bytes
+# directly to the content-addressed authority blob. Also prove all three hashes
+# (section, manifest, and raw bytes) agree.
+if node - "$NARRATIVE_DIR" "$ENVELOPE1" "$TMP" <<'NODE'
+const fs=require('fs'),c=require('crypto'); const dir=process.argv[2],env=JSON.parse(fs.readFileSync(process.argv[3])),out=process.argv[4],m=JSON.parse(fs.readFileSync(`${dir}/source-manifest.json`));
+for (const authority of ['flow','surface']) {
+  const section=env.sections.find(item=>item.authority===authority);
+  if (!section || typeof section.embedded_bytes!=='string') process.exit(1);
+  const entry=m.sources.find(item=>section.source_refs.includes(item.source_id));
+  if (!entry || entry.status!=='snapshotted') process.exit(1);
+  const raw=fs.readFileSync(`${dir}/sources/${entry.sha256}`);
+  const digest=c.createHash('sha256').update(raw).digest('hex');
+  if (section.sha256!==entry.sha256 || digest!==entry.sha256 || c.createHash('sha256').update(section.embedded_bytes).digest('hex')!==section.sha256) process.exit(1);
+  fs.writeFileSync(`${out}/${authority}.embedded`,section.embedded_bytes);
+  fs.writeFileSync(`${out}/${authority}.blob`,raw);
+}
+NODE
+then
+  if cmp -s "$TMP/flow.embedded" "$TMP/flow.blob" && cmp -s "$TMP/surface.embedded" "$TMP/surface.blob"; then
+    _pass "AC8: Flow and Surface embedded_bytes are byte-identical to frozen blobs"
+  else
+    _fail "AC8: extracted authority bytes differ from frozen blobs"
+  fi
+else _fail "AC8: foreign authority hash or embedded_bytes contract failed"; fi
+
+# Mutating a copied blob must be detected, then restored byte-identically.
+ENTRY_SHA="$(node -e "const m=JSON.parse(require('fs').readFileSync(process.argv[1]));process.stdout.write(m.sources.find(s=>s.status==='snapshotted').sha256)" "$NARRATIVE_DIR/source-manifest.json")"
+BLOB="$NARRATIVE_DIR/sources/$ENTRY_SHA"; cp "$BLOB" "$TMP/blob.backup"; printf 'mutation' >"$BLOB"
+if ! node "$ROOT/build/src/cli.js" narrative-sources verify --narrative-dir "$NARRATIVE_DIR" --json >"$TMP/mutated.json" 2>/dev/null; then _pass "AC2: post-compile blob mutation is detected"; else _fail "AC2: mutation passed verification"; fi
+cp "$TMP/blob.backup" "$BLOB"
+if cmp -s "$TMP/blob.backup" "$BLOB"; then _pass "AC2: mutated blob restored byte-identically"; else _fail "AC2: blob restoration differed"; fi
+
+if node "$ROOT/build/src/cli.js" narrative-sources resolve --narrative-dir "$NARRATIVE_DIR" --source-id "${SOURCES[3]}" --json >/dev/null; then _pass "AC9: offline resolver succeeds"; else _fail "AC9: offline resolver failed"; fi
+
+echo ""
+if [[ "$errors" -eq 0 ]]; then echo "narrative grounded envelope tests passed."; exit 0; fi
+echo "narrative grounded envelope tests FAILED: $errors issue(s)."
+exit 1

package/evals/integration/test_narrative_grounding_validator.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+# Deterministic grounding publication gate, adversarial corpus, and scorer-teeth proof (#623).
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+FIXTURES="$ROOT/evals/fixtures/narrative-grounding-validator"
+TMP="$(mktemp -d)"
+COMPILED="$ROOT/build/src/narrative/grounding-validator.js"
+BACKUP="$TMP/grounding-validator.js.original"
+errors=0
+
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+restore_compiled() {
+  if [[ -f "$BACKUP" ]]; then cp "$BACKUP" "$COMPILED"; fi
+}
+cleanup() { restore_compiled; rm -rf "$TMP"; }
+trap cleanup EXIT
+
+echo "Narrative grounding validator integration"
+if npm run build --silent; then _pass "TypeScript build completed"; else _fail "TypeScript build failed"; fi
+cp "$COMPILED" "$BACKUP"
+
+if node "$FIXTURES/scorer.mjs" all >"$TMP/scorer.out" 2>"$TMP/scorer.err"; then
+  _pass "AC1-AC4/AC6: adversarial corpus meets expected accept/reject verdicts"
+else
+  _fail "adversarial corpus failed: $(cat "$TMP/scorer.out" "$TMP/scorer.err")"
+fi
+if grep -q 'scorer counts: accept=2 reject=8 total=10' "$TMP/scorer.out" \
+  && grep -q 'unsupported observed-claim rate=0' "$TMP/scorer.out" \
+  && grep -q 'citation-resolver failures for published observed statements=0' "$TMP/scorer.out" \
+  && grep -q 'fixture provenance: kontourai/flow-agents#623' "$TMP/scorer.out"; then
+  _pass "AC6: scorer prints fixture provenance, counts, and enforces both zero thresholds"
+else
+  _fail "AC6 scorer report is incomplete: $(cat "$TMP/scorer.out")"
+fi
+if grep -q 'prompt-injection-inert: expected=accept actual=accept' "$TMP/scorer.out" \
+  && ! grep -q 'NGV_CANARY_623' "$TMP/scorer.out" "$TMP/scorer.err"; then
+  _pass "AC4: cited prompt injection leaves verdict/classes/source_refs byte-identical and canary-free"
+else
+  _fail "AC4 prompt-injection fixture changed grounded output or leaked its canary"
+fi
+
+# Build one publishable narrative and one automatically projected unavailable
+# citation. Both are composed through the real built CLI below.
+node --input-type=module - "$ROOT" "$TMP" <<'NODE'
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+const root=process.argv[2], tmp=process.argv[3];
+const api=await import(pathToFileURL(path.join(root,'build/src/index.js')));
+const sha256=bytes=>createHash('sha256').update(bytes).digest('hex');
+const compiler={name:'grounding-validator-integration',version:'1',policy_hash:'fixture'};
+const captureCompleteness={channels:{},known_gaps:[]};
+const repo=path.join(tmp,'repo'); fs.mkdirSync(repo,{recursive:true});
+const bytes=Buffer.from('{"kind":"created-file-fixture","status":"created"}\n');
+fs.writeFileSync(path.join(repo,'created.json'),bytes);
+api.snapshotNarrative({
+  narrativeDir:path.join(tmp,'valid-narrative'),narrativeId:'valid-cli',redactionFields:[],compiler,captureCompleteness,
+  requests:[{source:api.parseSourceId(`fa1:file:created.json:${sha256(bytes)}`),roots:{repoRoot:repo}}],
+},{now:()=> '2026-07-14T15:00:00.000Z'});
+api.snapshotNarrative({
+  narrativeDir:path.join(tmp,'unresolved-narrative'),narrativeId:'unresolved-cli',redactionFields:[],compiler,captureCompleteness,
+  requests:[{source:api.parseSourceId(`fa1:file:missing.json:${'0'.repeat(64)}`),roots:{repoRoot:repo}}],
+},{now:()=> '2026-07-14T15:00:00.000Z'});
+NODE
+
+if node "$ROOT/build/src/cli.js" narrative-sources compose \
+  --narrative-dir "$TMP/valid-narrative" --compiled-at "2026-07-14T16:00:00.000Z" \
+  --out-dir "$TMP/valid-out" >"$TMP/valid-compose.out" 2>"$TMP/valid-compose.err"; then
+  _pass "R1/R4: built CLI publishes a fully grounded narrative"
+else
+  _fail "valid built-CLI compose failed: $(cat "$TMP/valid-compose.err")"
+fi
+
+if node "$ROOT/build/src/cli.js" narrative-sources compose \
+  --narrative-dir "$TMP/unresolved-narrative" --compiled-at "2026-07-14T16:00:00.000Z" \
+  --out-dir "$TMP/unresolved-out" >"$TMP/unresolved-compose.out" 2>"$TMP/unresolved-compose.err"; then
+  _pass "completeness disclosure: manifest-declared unavailable source remains publishable"
+else
+  _fail "typed unavailable-source disclosure was incorrectly rejected: $(cat "$TMP/unresolved-compose.err")"
+fi
+
+# Couple the real AC1 scorer verdict to the real CLI publication boundary by
+# forcing that typed verdict at the stable branch anchor. The scorer above
+# proves the production citation branch creates this violation; this proves
+# compose refuses the same verdict before writeEnvelope.
+node - "$COMPILED" <<'NODE'
+const fs=require('fs'); const file=process.argv[2]; let text=fs.readFileSync(file,'utf8');
+const anchor='/* grounding-check:citation */';
+if (!text.includes(anchor)) throw new Error('citation mutation anchor missing');
+text=text.replace(anchor, `${anchor}\n    violations.push({ code: "unresolved_citation", statement_id: "forced", source_ref: "fa1:flow-state:absent:state/00000000", reason: "not_captured", detail: "forced integration verdict" });`);
+fs.writeFileSync(file,text);
+NODE
+if node --check "$COMPILED" >/dev/null \
+  && ! node "$ROOT/build/src/cli.js" narrative-sources compose \
+    --narrative-dir "$TMP/valid-narrative" --compiled-at "2026-07-14T16:00:00.000Z" \
+    --out-dir "$TMP/forced-violation-out" >"$TMP/forced-violation.out" 2>"$TMP/forced-violation.err" \
+  && grep -q 'narrative grounding gate failed: unresolved_citation' "$TMP/forced-violation.err" \
+  && [[ ! -e "$TMP/forced-violation-out" ]]; then
+  _pass "AC1/R4: typed grounding violation crosses the built CLI gate and writes no envelope"
+else
+  _fail "AC1 typed verdict did not fail closed at the CLI boundary: $(cat "$TMP/forced-violation.err" 2>/dev/null || true)"
+fi
+restore_compiled
+
+# A validator exception must remain upstream of writeEnvelope.
+node - "$COMPILED" <<'NODE'
+const fs=require('fs'); const file=process.argv[2]; let text=fs.readFileSync(file,'utf8');
+const anchor='/* grounding-check:citation */';
+if (!text.includes(anchor)) throw new Error('citation mutation anchor missing');
+text=text.replace(anchor, `${anchor}\n    throw new Error("AC5 injected validator failure");`);
+fs.writeFileSync(file,text);
+NODE
+if node --check "$COMPILED" >/dev/null \
+  && ! node "$ROOT/build/src/cli.js" narrative-sources compose \
+    --narrative-dir "$TMP/valid-narrative" --compiled-at "2026-07-14T16:00:00.000Z" \
+    --out-dir "$TMP/throw-out" >"$TMP/throw.out" 2>"$TMP/throw.err" \
+  && grep -q 'AC5 injected validator failure' "$TMP/throw.err" \
+  && [[ ! -e "$TMP/throw-out" ]]; then
+  _pass "AC5/R4: validator-internal exception fails closed before content-addressed write"
+else
+  _fail "AC5 validator exception did not fail closed: $(cat "$TMP/throw.err" 2>/dev/null || true)"
+fi
+restore_compiled
+cmp -s "$BACKUP" "$COMPILED" || _fail "compiled validator was not restored after AC5 mutation"
+
+mutation_test() {
+  local check="$1" anchor="$2" call="$3"
+  restore_compiled
+  node - "$COMPILED" "$anchor" "$call" <<'NODE'
+const fs=require('fs'); const [file,anchor,call]=process.argv.slice(2); let text=fs.readFileSync(file,'utf8');
+const needle=`${anchor}\n    ${call}`;
+if (!text.includes(needle)) throw new Error(`mutation target missing: ${needle}`);
+text=text.replace(needle, `${anchor}\n    // mutation: check deliberately disabled`);
+fs.writeFileSync(file,text);
+NODE
+  if ! node --check "$COMPILED" >/dev/null; then
+    _fail "${check} mutation produced invalid JavaScript"
+  elif node "$FIXTURES/scorer.mjs" "$check" >"$TMP/mutation-$check.out" 2>"$TMP/mutation-$check.err"; then
+    _fail "${check} scorer still passed after its validator branch was disabled"
+  elif grep -q 'expected=reject actual=accept' "$TMP/mutation-$check.out"; then
+    _pass "scorer teeth: disabling ${check} check flips reject to accept"
+  else
+    _fail "${check} mutation failed for an unrelated reason: $(cat "$TMP/mutation-$check.out" "$TMP/mutation-$check.err")"
+  fi
+  restore_compiled
+  if cmp -s "$BACKUP" "$COMPILED" && node "$FIXTURES/scorer.mjs" "$check" >/dev/null; then
+    _pass "${check} compiled branch restored byte-for-byte and corpus rejects again"
+  else
+    _fail "${check} compiled branch did not restore cleanly"
+  fi
+}
+
+mutation_test citation '/* grounding-check:citation */' 'violations.push(...citationViolations(statements, narrativeDir, resolver, declaredUnavailable));'
+mutation_test material '/* grounding-check:material */' 'violations.push(...materialViolations(deriveMaterialEvents(manifest, resolved), statements));'
+mutation_test epistemic '/* grounding-check:epistemic */' 'violations.push(...epistemicViolations(statements));'
+
+echo ""
+if [[ "$errors" -eq 0 ]]; then
+  echo "narrative grounding validator tests passed: 14/14."
+  exit 0
+fi
+echo "narrative grounding validator tests FAILED: $errors issue(s)."
+exit 1