@kontourai/flow-agents 3.12.1 → 4.1.0

package/dist/base/evals/integration/test_telemetry_tool_outcome.sh

@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+# test_telemetry_tool_outcome.sh — Layer 2: per-tool-result enrichment
+# (kontourai/flow-agents#580, split from #568)
+#
+# Proves that tool.result (postToolUse/PostToolUse/PostToolUseFailure) telemetry
+# events carry three new fields on their .tool object:
+#   - .tool.duration_ms — non-negative wall-clock ms between this tool's invoke
+#     and its result (correlated per tool call), or null when the matching start
+#     record is absent (never a fabricated 0 / stale value).
+#   - .tool.outcome — a deterministic tri-state pass|fail|ambiguous, a jq port
+#     of scripts/hooks/evidence-capture.js observeResult (never from stdout text).
+#   - .tool.status — the host exit code int when present, else null.
+#
+# The highest-value assertion is the DRIFT-GUARD (case j): a shared fixture
+# battery is fed through BOTH the emitter jq path AND node observeResult, and
+# their {exitCode, observedResult} verdicts must be identical — so the hermetic
+# jq re-implementation can never silently diverge from the canonical source.
+#
+# Uses the same TELEMETRY_DIR resolution and FLOW_AGENTS_TELEMETRY_FOREGROUND
+# convention as test_telemetry_tool_usage.sh so assertions never race a
+# backgrounded subshell.
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+if [[ -d "$ROOT_DIR/context/scripts/telemetry" ]]; then
+  TELEMETRY_DIR="$ROOT_DIR/context/scripts/telemetry"
+else
+  TELEMETRY_DIR="<bundle-root>/context/scripts/telemetry"
+fi
+TELEMETRY_SH="${TELEMETRY_DIR}/telemetry.sh"
+SOURCE_TELEMETRY_SH="$ROOT_DIR/scripts/telemetry/telemetry.sh"
+FIXTURE_TRANSCRIPT="$ROOT_DIR/evals/fixtures/telemetry/usage-transcript-sample.jsonl"
+PRICING_FILE="$ROOT_DIR/scripts/telemetry/pricing.json"
+EVIDENCE_CAPTURE="$ROOT_DIR/scripts/hooks/evidence-capture.js"
+
+TMPDIR_EVAL=$(mktemp -d /tmp/eval-telemetry-tool-outcome.XXXXXX)
+TMPLOG="${TMPDIR_EVAL}/test-output.jsonl"
+
+pass=0; fail=0
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+
+echo "=== Layer 2: Telemetry Tool-Result Enrichment (#580: duration/outcome/status) ==="
+echo ""
+
+if [[ ! -f "$TELEMETRY_SH" ]]; then
+  _fail "telemetry.sh not found at $TELEMETRY_SH"
+  rm -rf "$TMPDIR_EVAL"; exit 1
+fi
+if [[ ! -f "$EVIDENCE_CAPTURE" ]]; then
+  _fail "evidence-capture.js not found at $EVIDENCE_CAPTURE (needed for drift guard)"
+  rm -rf "$TMPDIR_EVAL"; exit 1
+fi
+
+# Run one hook event against telemetry.sh (foreground) and return the resulting
+# event object (jq-compact, one line) from the full-channel log.
+_run_tool_event() {
+  local hook_type="$1" input="$2"; shift 2
+  local extra_env=("$@")
+  local common_env=(
+    HOME="${TMPDIR_EVAL}/home"
+    TELEMETRY_ENABLED=true
+    TELEMETRY_CHANNELS=full
+    TELEMETRY_CHANNEL_FULL_LOG_FILE="$TMPLOG"
+    FLOW_AGENTS_TELEMETRY_FOREGROUND=true
+    TELEMETRY_CONFIG_FILE="$TMPDIR_EVAL/telemetry.conf"
+    TELEMETRY_DATA_DIR="$TMPDIR_EVAL"
+    TELEMETRY_SESSION_DIR="$TMPDIR_EVAL/sessions"
+    TELEMETRY_USAGE_TRACKING=true
+  )
+  mkdir -p "${TMPDIR_EVAL}/home" "${TMPDIR_EVAL}/sessions"
+
+  local before_lines
+  touch "$TMPLOG"
+  before_lines=$(wc -l < "$TMPLOG" | tr -d ' ')
+
+  echo "$input" | env "${common_env[@]}" ${extra_env[@]+"${extra_env[@]}"} bash "${RUN_TELEMETRY_SH:-$TELEMETRY_SH}" "$hook_type" dev 2>/dev/null
+
+  tail -n +"$((before_lines + 1))" "$TMPLOG" 2>/dev/null | tail -1
+}
+
+# --- (a) invoke -> result: duration_ms is a non-negative number ---------------
+echo "--- (a) invoke->result duration_ms is a non-negative number ---"
+inv_a=$(jq -nc '{session_id:"dur-a",hook_event_name:"PreToolUse",tool_use_id:"tu-a",tool_name:"Bash",tool_input:{command:"echo hi"}}')
+_run_tool_event "preToolUse" "$inv_a" >/dev/null
+res_a=$(jq -nc '{session_id:"dur-a",hook_event_name:"PostToolUse",tool_use_id:"tu-a",tool_name:"Bash",tool_input:{command:"echo hi"},tool_response:{exitCode:0}}')
+out_a=$(_run_tool_event "postToolUse" "$res_a")
+dur_a=$(echo "$out_a" | jq -r '.tool.duration_ms')
+if [[ "$dur_a" =~ ^[0-9]+$ ]]; then
+  _pass "duration_ms is a non-negative integer ($dur_a) when the matching invoke was recorded"
+else
+  _fail "expected numeric duration_ms, got '$dur_a'"
+fi
+
+# --- (b) result with NO prior invoke -> duration_ms is exactly null -----------
+echo ""
+echo "--- (b) result with no prior invoke -> duration_ms:null (never 0/stale) ---"
+res_b=$(jq -nc '{session_id:"dur-b",hook_event_name:"PostToolUse",tool_use_id:"tu-b-orphan",tool_name:"Bash",tool_input:{command:"echo hi"},tool_response:{exitCode:0}}')
+out_b=$(_run_tool_event "postToolUse" "$res_b")
+dur_b=$(echo "$out_b" | jq -r '.tool.duration_ms')
+[[ "$dur_b" == "null" ]] && _pass "duration_ms is exactly null (not 0) when no invoke start record exists" || _fail "expected duration_ms=null, got '$dur_b'"
+
+# --- (c) exit 0 -> outcome:pass status:0 -------------------------------------
+echo ""
+echo "--- (c) host exit code 0 -> outcome:pass status:0 ---"
+res_c=$(jq -nc '{session_id:"oc-c",hook_event_name:"PostToolUse",tool_use_id:"tu-c",tool_name:"Bash",tool_input:{command:"echo hi"},tool_response:{exitCode:0}}')
+out_c=$(_run_tool_event "postToolUse" "$res_c")
+oc_c=$(echo "$out_c" | jq -r '.tool.outcome'); st_c=$(echo "$out_c" | jq -r '.tool.status')
+[[ "$oc_c" == "pass" ]] && _pass "outcome=pass on exit 0" || _fail "expected outcome=pass, got '$oc_c'"
+[[ "$st_c" == "0" ]] && _pass "status=0 on exit 0" || _fail "expected status=0, got '$st_c'"
+
+# --- (d) non-zero exit -> outcome:fail status:<code> -------------------------
+echo ""
+echo "--- (d) non-zero host exit code -> outcome:fail status:<code> ---"
+res_d=$(jq -nc '{session_id:"oc-d",hook_event_name:"PostToolUse",tool_use_id:"tu-d",tool_name:"Bash",tool_input:{command:"false"},tool_response:{exitCode:2}}')
+out_d=$(_run_tool_event "postToolUse" "$res_d")
+oc_d=$(echo "$out_d" | jq -r '.tool.outcome'); st_d=$(echo "$out_d" | jq -r '.tool.status')
+[[ "$oc_d" == "fail" ]] && _pass "outcome=fail on exit 2" || _fail "expected outcome=fail, got '$oc_d'"
+[[ "$st_d" == "2" ]] && _pass "status=2 carries the real exit code" || _fail "expected status=2, got '$st_d'"
+
+# --- (e) failure signal, NO clean code -> outcome:fail status:null -----------
+echo ""
+echo "--- (e) is_error:true / success:false with no exit code -> fail, status:null ---"
+res_e1=$(jq -nc '{session_id:"oc-e1",hook_event_name:"PostToolUse",tool_use_id:"tu-e1",tool_name:"Bash",tool_input:{command:"x"},tool_response:{is_error:true,output:"oops"}}')
+out_e1=$(_run_tool_event "postToolUse" "$res_e1")
+oc_e1=$(echo "$out_e1" | jq -r '.tool.outcome'); st_e1=$(echo "$out_e1" | jq -r '.tool.status')
+[[ "$oc_e1" == "fail" ]] && _pass "outcome=fail on is_error:true (no code)" || _fail "expected fail, got '$oc_e1'"
+[[ "$st_e1" == "null" ]] && _pass "status=null when no clean exit code is present (never fabricated)" || _fail "expected status=null, got '$st_e1'"
+res_e2=$(jq -nc '{session_id:"oc-e2",hook_event_name:"PostToolUse",tool_use_id:"tu-e2",tool_name:"Bash",tool_input:{command:"x"},tool_response:{success:false}}')
+out_e2=$(_run_tool_event "postToolUse" "$res_e2")
+oc_e2=$(echo "$out_e2" | jq -r '.tool.outcome')
+[[ "$oc_e2" == "fail" ]] && _pass "outcome=fail on success:false (no code)" || _fail "expected fail, got '$oc_e2'"
+
+# --- (f) no signal at all -> outcome:ambiguous status:null -------------------
+echo ""
+echo "--- (f) no exit code and no failure signal -> ambiguous, status:null ---"
+res_f=$(jq -nc '{session_id:"oc-f",hook_event_name:"PostToolUse",tool_use_id:"tu-f",tool_name:"Read",tool_input:{file:"a"},tool_response:{output:"contents"}}')
+out_f=$(_run_tool_event "postToolUse" "$res_f")
+oc_f=$(echo "$out_f" | jq -r '.tool.outcome'); st_f=$(echo "$out_f" | jq -r '.tool.status')
+[[ "$oc_f" == "ambiguous" ]] && _pass "outcome=ambiguous when there is no positive success evidence (never pass)" || _fail "expected ambiguous, got '$oc_f'"
+[[ "$st_f" == "null" ]] && _pass "status=null in the no-signal case" || _fail "expected status=null, got '$st_f'"
+
+# --- (g) PostToolUseFailure event folds to fail -----------------------------
+echo ""
+echo "--- (g) PostToolUseFailure event folds to outcome:fail ---"
+res_g=$(jq -nc '{session_id:"oc-g",hook_event_name:"PostToolUseFailure",tool_use_id:"tu-g",tool_name:"Bash",tool_input:{command:"x"},tool_response:{output:"partial"}}')
+out_g=$(_run_tool_event "PostToolUseFailure" "$res_g")
+et_g=$(echo "$out_g" | jq -r '.event_type'); oc_g=$(echo "$out_g" | jq -r '.tool.outcome')
+[[ "$et_g" == "tool.result" ]] && _pass "PostToolUseFailure maps to tool.result" || _fail "expected tool.result, got '$et_g'"
+[[ "$oc_g" == "fail" ]] && _pass "PostToolUseFailure folds to outcome=fail regardless of payload signal" || _fail "expected fail, got '$oc_g'"
+
+# --- (h) invoke and permission_request carry NO duration/outcome/status ------
+echo ""
+echo "--- (h) tool.invoke and tool.permission_request carry no result meta ---"
+inv_h=$(jq -nc '{session_id:"h-inv",hook_event_name:"PreToolUse",tool_use_id:"tu-h",tool_name:"Read",tool_input:{file:"a"}}')
+out_h_inv=$(_run_tool_event "preToolUse" "$inv_h")
+has_dur_inv=$(echo "$out_h_inv" | jq -r '.tool | has("duration_ms")')
+has_oc_inv=$(echo "$out_h_inv" | jq -r '.tool | has("outcome")')
+[[ "$has_dur_inv" == "false" && "$has_oc_inv" == "false" ]] && _pass "tool.invoke has no duration_ms/outcome/status (invoke has no result yet)" || _fail "tool.invoke unexpectedly carries result meta (duration=$has_dur_inv outcome=$has_oc_inv)"
+perm_h=$(jq -nc '{session_id:"h-perm",hook_event_name:"PermissionRequest",tool_use_id:"tu-hp",tool_name:"Bash",tool_input:{command:"x",description:"run"}}')
+out_h_perm=$(_run_tool_event "permissionRequest" "$perm_h")
+has_oc_perm=$(echo "$out_h_perm" | jq -r '.tool | has("outcome")')
+[[ "$has_oc_perm" == "false" ]] && _pass "tool.permission_request carries no outcome (not a tool result)" || _fail "tool.permission_request unexpectedly carries an outcome"
+
+# --- (i) slice-1 .usage still present and correct (no regression) ------------
+echo ""
+echo "--- (i) slice-1 .usage enrichment still present on tool.result (no regression) ---"
+res_i=$(jq -nc --arg tp "$FIXTURE_TRANSCRIPT" '{session_id:"i-usage",transcript_path:$tp,hook_event_name:"PostToolUse",tool_use_id:"tu-i",tool_name:"Bash",tool_input:{command:"echo hi"},tool_response:{exitCode:0}}')
+out_i=$(_run_tool_event "postToolUse" "$res_i" TELEMETRY_PRICING_FILE="$PRICING_FILE")
+model_i=$(echo "$out_i" | jq -r '.usage.model // empty'); oc_i=$(echo "$out_i" | jq -r '.tool.outcome')
+[[ "$model_i" == "claude-fable-5" ]] && _pass "usage.model is still the last-turn model (claude-fable-5) alongside the new fields" || _fail "slice-1 regression: expected usage.model=claude-fable-5, got '$model_i'"
+[[ "$oc_i" == "pass" ]] && _pass "new outcome field coexists with slice-1 .usage on the same record" || _fail "expected outcome=pass, got '$oc_i'"
+
+# --- (j) DRIFT-GUARD: emitter jq outcome == node observeResult across battery -
+echo ""
+echo "--- (j) DRIFT-GUARD: emitter jq outcome/status == node observeResult (shared battery) ---"
+# Shared fixture battery: [name, tool_input_json_or_null, tool_response_json_or_null, top_level_extra_json]
+# Each row is fed BOTH through the full emitter (postToolUse) and node observeResult.
+battery_json=$(cat <<'JSON'
+[
+  {"name":"exit0","ti":{"command":"echo hi"},"tr":{"exitCode":0},"extra":{}},
+  {"name":"exit2","ti":{"command":"false"},"tr":{"exitCode":2},"extra":{}},
+  {"name":"str_code_0","ti":{"command":"x"},"tr":{"exit_code":"0"},"extra":{}},
+  {"name":"is_error","ti":{"command":"x"},"tr":{"is_error":true},"extra":{}},
+  {"name":"success_false","ti":{"command":"x"},"tr":{"success":false},"extra":{}},
+  {"name":"error_top","ti":{"command":"x"},"tr":{"output":"ok"},"extra":{"error":"boom"}},
+  {"name":"error_top_array","ti":{"command":"x"},"tr":{"output":"ok"},"extra":{"error":["boom"]}},
+  {"name":"no_signal","ti":{"file":"a"},"tr":{"output":"contents"},"extra":{}},
+  {"name":"grep_absence","ti":{"command":"grep -r foo src"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"grep_or_true","ti":{"command":"grep -r foo src || true"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"grep_negated","ti":{"command":"! grep foo x"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"diff_absence","ti":{"command":"diff a b"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"diff_err2","ti":{"command":"diff a b"},"tr":{"exitCode":2},"extra":{}},
+  {"name":"bashlc_grep","ti":{"command":"bash -lc 'grep foo x'"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"keyval_grep","ti":{"command":"FOO=bar grep foo x"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"piped_grep","ti":{"command":"grep foo x | wc -l"},"tr":{"exitCode":1},"extra":{}},
+  {"name":"returnCode0","ti":{"command":"x"},"tr":{"returnCode":0},"extra":{}},
+  {"name":"top_status1","ti":{"command":"x"},"tr":{"output":"noise"},"extra":{"status":1}},
+  {"name":"stderr_nostdout","ti":{"command":"x"},"tr":{"stderr":"boom"},"extra":{}},
+  {"name":"stderr_withstdout","ti":{"command":"x"},"tr":{"stderr":"boom","stdout":"ok"},"extra":{}},
+  {"name":"plain_string_out","ti":{"command":"x"},"tr":"just a string","extra":{}}
+]
+JSON
+)
+battery_count=$(echo "$battery_json" | jq 'length')
+drift_ok=0; drift_bad=0
+while IFS= read -r fixture; do
+  bname=$(echo "$fixture" | jq -r '.name')
+  # Build the hook payload (unique tool_use_id so no cross-case duration leak).
+  payload=$(echo "$fixture" | jq -c '{session_id:("drift-"+.name),hook_event_name:"PostToolUse",tool_use_id:("tu-drift-"+.name),tool_name:"Bash",tool_input:.ti,tool_response:.tr} + .extra')
+  # Emitter path.
+  emit_out=$(_run_tool_event "postToolUse" "$payload")
+  emit_oc=$(echo "$emit_out" | jq -r '.tool.outcome')
+  emit_st=$(echo "$emit_out" | jq -c '.tool.status')
+  # Node canonical path — mirror observeResult's exact call shape at the run() site.
+  node_json=$(EVCAP="$EVIDENCE_CAPTURE" PAYLOAD="$payload" node -e '
+    const {observeResult}=require(process.env.EVCAP);
+    const i=JSON.parse(process.env.PAYLOAD);
+    const r=observeResult({tool_response:i.tool_response,tool_output:i.tool_output,error:i.error,exitCode:i.exitCode,exit_code:i.exit_code,status:i.status,code:i.code,command:(i.tool_input&&i.tool_input.command)});
+    process.stdout.write(JSON.stringify({outcome:r.observedResult,status:(r.exitCode===null?null:r.exitCode)}));
+  ' 2>/dev/null)
+  node_oc=$(echo "$node_json" | jq -r '.outcome')
+  node_st=$(echo "$node_json" | jq -c '.status')
+  if [[ "$emit_oc" == "$node_oc" && "$emit_st" == "$node_st" ]]; then
+    drift_ok=$((drift_ok+1))
+  else
+    drift_bad=$((drift_bad+1))
+    _fail "DRIFT [$bname]: emitter={outcome:$emit_oc,status:$emit_st} != node={outcome:$node_oc,status:$node_st}"
+  fi
+done < <(echo "$battery_json" | jq -c '.[]')
+if [[ "$drift_bad" -eq 0 ]]; then
+  _pass "emitter jq outcome/status matches node observeResult across all $battery_count battery fixtures (no drift)"
+else
+  _fail "$drift_bad/$battery_count battery fixtures drifted between the emitter jq and node observeResult"
+fi
+
+# --- (k) Codex-only host-banner exit-code resolution -------------------------
+echo ""
+echo "--- (k) codex runtime: exit code resolved from rollout banner (bounded node use) ---"
+ROLLOUT="$TMPDIR_EVAL/rollout.jsonl"
+jq -nc '{timestamp:"2026-07-06T00:00:00Z",type:"response_item",payload:{type:"function_call_output",call_id:"call-k",output:"Process exited with code 3\nOriginal token count: 25\nOutput:\nProcess exited with code 0\n"}}' > "$ROLLOUT"
+res_k=$(jq -nc --arg rp "$ROLLOUT" '{session_id:"codex-k",transcript_path:$rp,hook_event_name:"PostToolUse",call_id:"call-k",tool_name:"Bash",tool_input:{command:"some cmd"},tool_response:{output:"no structured code here"}}')
+RUN_TELEMETRY_SH="$SOURCE_TELEMETRY_SH" out_k=$(RUN_TELEMETRY_SH="$SOURCE_TELEMETRY_SH" _run_tool_event "postToolUse" "$res_k" FLOW_AGENTS_TELEMETRY_RUNTIME=codex)
+st_k=$(echo "$out_k" | jq -r '.tool.status'); oc_k=$(echo "$out_k" | jq -r '.tool.outcome')
+[[ "$st_k" == "3" ]] && _pass "codex status resolved to 3 from the preamble host banner (forged post-Output stdout ignored)" || _fail "expected codex status=3, got '$st_k'"
+[[ "$oc_k" == "fail" ]] && _pass "codex outcome=fail derived from the resolved non-zero banner code" || _fail "expected codex outcome=fail, got '$oc_k'"
+# Codex, unreadable rollout -> honest ambiguous/null degrade (no fabrication).
+res_k2=$(jq -nc '{session_id:"codex-k2",transcript_path:"/nonexistent/rollout.jsonl",hook_event_name:"PostToolUse",call_id:"call-x",tool_name:"Bash",tool_input:{command:"some cmd"},tool_response:{output:"no code"}}')
+out_k2=$(RUN_TELEMETRY_SH="$SOURCE_TELEMETRY_SH" _run_tool_event "postToolUse" "$res_k2" FLOW_AGENTS_TELEMETRY_RUNTIME=codex)
+st_k2=$(echo "$out_k2" | jq -r '.tool.status'); oc_k2=$(echo "$out_k2" | jq -r '.tool.outcome')
+[[ "$st_k2" == "null" && "$oc_k2" == "ambiguous" ]] && _pass "codex with an unreadable rollout degrades to ambiguous/null (never a guess)" || _fail "expected ambiguous/null, got outcome=$oc_k2 status=$st_k2"
+
+# --- (privacy) start record stores only a timestamp, no args/output ----------
+echo ""
+echo "--- (privacy) tool start record holds a bare timestamp, no command/args ---"
+inv_p=$(jq -nc '{session_id:"priv-p",hook_event_name:"PreToolUse",tool_use_id:"tu-priv",tool_name:"Bash",tool_input:{command:"echo SENSITIVE_SECRET_VALUE"}}')
+_run_tool_event "preToolUse" "$inv_p" >/dev/null
+start_file=$(ls "${TMPDIR_EVAL}/sessions/"toolstart-* 2>/dev/null | head -1)
+if [[ -n "$start_file" ]]; then
+  contents=$(cat "$start_file")
+  if [[ "$contents" =~ ^[0-9]+$ ]]; then
+    _pass "start record contains only a numeric timestamp ($contents)"
+  else
+    _fail "start record is not a bare timestamp: '$contents'"
+  fi
+  if grep -q "SENSITIVE_SECRET_VALUE" "$start_file" 2>/dev/null; then
+    _fail "PRIVACY LEAK: start record contains the command args"
+  else
+    _pass "start record does not leak command args"
+  fi
+else
+  _fail "no tool start record written on preToolUse"
+fi
+
+rm -rf "$TMPDIR_EVAL"
+
+echo ""
+echo "Telemetry tool-result enrichment: $pass passed, $fail failed"
+[ "$fail" -eq 0 ]

package/dist/base/evals/integration/test_veritas_governance_kit.sh

@@ -133,6 +133,72 @@ echo "--- exemption-issuance: gate blocks without a verified approval, passes wi
 exemption_gate_case "$KIT/fixtures/exemption/approved.trust-bundle.json"     "exemption-positive" 0
 exemption_gate_case "$KIT/fixtures/exemption/not-approved.trust-bundle.json" "exemption-negative" 1
 
+# --- standards-authoring flow (Slice 2 PR3): registration, gate shape, gate cases ---
+# Agentless propose -> apply flow whose human-approval-gate blocks the `veritas init --apply`
+# write until a human-authored standards-authoring-approval claim is verified. The kit only
+# gates the sign-off; Veritas does the derivation and the write (no evaluation reimplemented).
+AUTHORING_FLOWDEF="$KIT/flows/standards-authoring.flow.json"
+if node -e "const f=require('$AUTHORING_FLOWDEF'); if(f.id!=='veritas-governance.standards-authoring') process.exit(1);" 2>/dev/null; then
+  pass "standards-authoring flow file has id veritas-governance.standards-authoring"
+else
+  fail "standards-authoring flow file missing or has unexpected id"
+fi
+if rg -q '"id": *"veritas-governance\.standards-authoring"' "$KIT/kit.json" && rg -q '"path": *"flows/standards-authoring\.flow\.json"' "$KIT/kit.json"; then
+  pass "standards-authoring flow is registered in kit.json flows[]"
+else
+  fail "standards-authoring flow is NOT registered in kit.json flows[]"
+fi
+if rg -q '"id": *"veritas-governance\.standards-authoring"' "$KIT/kit.json" && rg -q '"path": *"skills/standards-authoring/SKILL.md"' "$KIT/kit.json"; then
+  pass "standards-authoring skill is registered in kit.json skills[]"
+else
+  fail "standards-authoring skill is NOT registered in kit.json skills[]"
+fi
+# Agentless invariant: the kit declares no flow_step_actions (would opt into Builder's
+# producer-ownership contract and break the agentless gate flows).
+if rg -q 'flow_step_actions' "$KIT/kit.json"; then
+  fail "kit.json declares flow_step_actions — veritas-governance flows are agentless"
+else
+  pass "kit stays agentless (no flow_step_actions)"
+fi
+if node -e "
+const f=require('$AUTHORING_FLOWDEF');
+const g=f.gates['human-approval-gate'];
+const bc=g.expects[0].bundle_claim;
+if (g.expects[0].kind!=='trust.bundle') process.exit(1);
+if (bc.claimType!=='standards-authoring-approval') process.exit(1);
+if (bc.subjectType!=='repo-governance-change') process.exit(1);
+if (JSON.stringify(bc.accepted_statuses)!=='[\"verified\"]') process.exit(1);
+" 2>/dev/null; then
+  pass "standards-authoring gate expects[] kind is trust.bundle and pins claimType/subjectType/accepted_statuses exactly"
+else
+  fail "standards-authoring gate expects[] shape does not match the pinned claim selector"
+fi
+# Positive/negative gate cases against the committed fixtures. $1 bundle, $2 label, $3 expect.
+authoring_gate_case() {
+  local bundle="$1" label="$2" expect="$3"
+  local work="$TMP_DIR/$label"; mkdir -p "$work"; ( cd "$work" && node "$FLOW_CLI" init >/dev/null 2>&1 )
+  ( cd "$work" && node "$FLOW_CLI" start "$AUTHORING_FLOWDEF" --run-id "$label" >/dev/null 2>&1 )
+  ( cd "$work" && node "$FLOW_CLI" attach-evidence "$label" --gate human-approval-gate --file "$bundle" --bundle >"$work/attach.out" 2>&1 )
+  if ! rg -q 'kind: trust.bundle' "$work/attach.out"; then
+    fail "[$label] evidence did not attach as kind: trust.bundle"; sed -n '1,20p' "$work/attach.out"; return
+  fi
+  ( cd "$work" && node "$FLOW_CLI" evaluate "$label" --gate human-approval-gate --exit-code >"$work/eval.out" 2>&1 )
+  local got=$?
+  if [[ "$got" == "$expect" ]]; then
+    pass "[$label] standards-authoring gate evaluate exit $got as expected"
+  else
+    fail "[$label] standards-authoring gate evaluate exit $got, expected $expect"; sed -n '1,20p' "$work/eval.out"
+  fi
+}
+authoring_gate_case "$KIT/fixtures/standards-authoring/approved.trust-bundle.json"     "authoring-positive" 0
+authoring_gate_case "$KIT/fixtures/standards-authoring/not-approved.trust-bundle.json" "authoring-negative" 1
+# No-fork: the authoring skill wraps the veritas CLI; it must not vendor rule/claim evaluation.
+if rg -q -i 'evaluateRepoStandards|evidence-check-runner\.mjs|class +[A-Za-z]*RuleEngine|repo-standards/default\.repo-standards\.json' "$KIT/skills/standards-authoring" "$KIT/flows/standards-authoring.flow.json" 2>/dev/null; then
+  fail "standards-authoring skill/flow appears to vendor Veritas evaluation logic (no-fork violated)"
+else
+  pass "no-fork: standards-authoring wraps the veritas CLI, no vendored rule/claim engine"
+fi
+
 # --- exemption-issuance: issue-step DECLARED append semantics (AC4) --------------
 # Seed a scratch delivery/DECLARED with main's real 2-entry array (inlined here, not read
 # live from the repo, so this eval is not coupled to that file's future contents), then
@@ -364,6 +430,196 @@ else
   echo "  - SKIP live Veritas leg (set VERITAS_BIN + VERITAS_GOVERNED_REPO to enable); fixtures are captured real output"
 fi
 
+# --- Starter-standards provisioning (Slice 2 PR2): the kit scaffolds a runnable .veritas/ ---
+# The kit declares its starter Repo Standards as provisions[]; `kit provision` copies them into
+# a consumer repo. This asserts the scaffold lands and is well-formed; when a Veritas binary is
+# available it further asserts `veritas readiness` runs end-to-end against the scaffolded repo.
+echo "--- starter-standards provisioning ---"
+FA_CLI="$ROOT/build/src/cli.js"
+if [[ ! -f "$FA_CLI" ]]; then
+  fail "flow-agents CLI not built at build/src/cli.js (run npm run build)"
+else
+  prov="$TMP_DIR/provisioned-repo"; mkdir -p "$prov"
+  if node "$FA_CLI" kit provision "$KIT" --target "$prov" >"$TMP_DIR/provision.out" 2>&1; then
+    pass "kit provision scaffolds starter standards into a consumer repo"
+  else
+    fail "kit provision failed"; sed -n '1,20p' "$TMP_DIR/provision.out"
+  fi
+  # Every declared provision target must land and (for JSON) parse.
+  starter_targets=(
+    ".veritas/repo-map.json"
+    ".veritas/repo-standards/default.repo-standards.json"
+    ".veritas/authority/default.authority-settings.json"
+    ".veritas/GOVERNANCE.md"
+    ".veritas/README.md"
+    "veritas.claims.json"
+  )
+  for rel in "${starter_targets[@]}"; do
+    if [[ -f "$prov/$rel" ]]; then
+      if [[ "$rel" == *.json ]]; then
+        if node -e "JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))" "$prov/$rel" >/dev/null 2>&1; then
+          pass "scaffolded $rel is present and valid JSON"
+        else
+          fail "scaffolded $rel is not valid JSON"
+        fi
+      else
+        pass "scaffolded $rel is present"
+      fi
+    else
+      fail "expected scaffolded file missing: $rel"
+    fi
+  done
+  # The scaffold must reference the Veritas engine's OWN starter output, not reimplement it:
+  # every provision source lives under assets/, never in adapter/ or flows/ (no-fork already
+  # scopes there). Assert the declared sources exist inside the kit's assets tree.
+  if node -e "
+    const fs=require('fs'),path=require('path');
+    const kit=JSON.parse(fs.readFileSync(process.argv[1]+'/kit.json','utf8'));
+    const bad=(kit.provisions||[]).filter(p=>!p.path.startsWith('assets/')||!fs.existsSync(path.join(process.argv[1],p.path)));
+    if(bad.length){console.error('bad provisions:',bad.map(p=>p.id).join(','));process.exit(1)}
+  " "$KIT" >/dev/null 2>&1; then
+    pass "all starter provisions source from the kit's assets/ tree"
+  else
+    fail "a starter provision points outside assets/ or at a missing file"
+  fi
+  # Live: a Veritas binary must be able to run readiness against the scaffolded repo.
+  VBIN2="${VERITAS_BIN:-veritas}"
+  if command -v "$VBIN2" >/dev/null 2>&1 || [[ -x "$VBIN2" ]]; then
+    ( cd "$prov" && git init -q && printf '{"name":"scaffold-eval","scripts":{"test":"node -e \"process.exit(0)\""}}\n' > package.json \
+        && git add -A && git -c user.email=eval@local -c user.name=eval commit -qm scaffold ) >/dev/null 2>&1
+    if ( cd "$prov" && "$VBIN2" readiness --working-tree >/dev/null 2>&1 ); then :; fi
+    if ls "$prov/.kontourai/veritas/evidence/"*.json >/dev/null 2>&1; then
+      pass "veritas readiness runs end-to-end on the kit-scaffolded repo (evidence report produced)"
+    else
+      fail "veritas readiness produced no evidence report on the scaffolded repo"
+    fi
+  else
+    echo "  - SKIP scaffold readiness leg (no Veritas binary; set VERITAS_BIN to enable)"
+  fi
+fi
+
+# --- Governance git-hook provisioning (Slice 3 PR / #648) --------------------------
+# The kit ships the two governance git hooks as provisions[]; `kit provision` lands them
+# (non-executable), and a documented chmod + core.hooksPath step activates them. This asserts
+# the hook files land with the expected bodies and that, once activated, the pre-push hook fires.
+echo "--- governance git-hook provisioning ---"
+FA_CLI2="$ROOT/build/src/cli.js"
+if [[ ! -f "$FA_CLI2" ]]; then
+  fail "flow-agents CLI not built at build/src/cli.js (run npm run build)"
+else
+  hookrepo="$TMP_DIR/hook-provisioned-repo"; mkdir -p "$hookrepo"
+  ( cd "$hookrepo" && git init -q && printf '{"name":"hook-eval","scripts":{"prepush":"echo PREPUSH-RAN"}}\n' > package.json )
+  if node "$FA_CLI2" kit provision "$KIT" --target "$hookrepo" >"$TMP_DIR/hookprov.out" 2>&1; then
+    pass "kit provision lands the governance git hooks"
+  else
+    fail "kit provision (hooks) failed"; sed -n '1,20p' "$TMP_DIR/hookprov.out"
+  fi
+  # Both hook files present, match the committed kit assets byte-for-byte, AND carry their
+  # defining command. The byte-match alone cannot catch a corrupted kit asset (a bad asset is
+  # copied verbatim and still matches itself), so a content assertion pins what each hook must do.
+  declare -A hook_must_contain=(
+    [pre-push]='npm run --if-present prepush'
+    [post-commit]='veritas readiness --changed-from HEAD~1 --changed-to HEAD'
+  )
+  for hook in pre-push post-commit; do
+    if [[ -f "$hookrepo/.githooks/$hook" ]]; then
+      if cmp -s "$hookrepo/.githooks/$hook" "$KIT/assets/starter-hooks/githooks/$hook"; then
+        pass "scaffolded .githooks/$hook matches the kit asset byte-for-byte"
+      else
+        fail "scaffolded .githooks/$hook differs from the kit asset"
+      fi
+      if grep -qF "${hook_must_contain[$hook]}" "$hookrepo/.githooks/$hook"; then
+        pass "scaffolded .githooks/$hook invokes its defining command (${hook_must_contain[$hook]})"
+      else
+        fail "scaffolded .githooks/$hook is missing its defining command (${hook_must_contain[$hook]}) — corrupted asset?"
+      fi
+    else
+      fail "expected git hook missing: .githooks/$hook"
+    fi
+  done
+  # Provisions land non-executable by design (the engine copy is agent-blind) — assert for BOTH.
+  for hook in pre-push post-commit; do
+    if [[ -x "$hookrepo/.githooks/$hook" ]]; then
+      fail "provisioned .githooks/$hook is already executable — provisioning should be agent-blind (no +x)"
+    else
+      pass "provisioned .githooks/$hook lands non-executable (agent-blind copy; activation is explicit)"
+    fi
+  done
+  ( cd "$hookrepo" && chmod +x .githooks/pre-push .githooks/post-commit && git config core.hooksPath .githooks )
+  if [[ "$(cd "$hookrepo" && git config --get core.hooksPath)" == ".githooks" ]]; then
+    pass "activation sets core.hooksPath to .githooks"
+  else
+    fail "activation did not set core.hooksPath"
+  fi
+  # Prove GIT actually dispatches the hook via core.hooksPath — not just direct invocation. Push
+  # to a local bare remote; the pre-push hook (runs `npm run --if-present prepush`) fires through
+  # git's own hook machinery. Output captured to a file so pipefail + git's SIGPIPE can't interfere.
+  ( cd "$hookrepo" && git -c user.email=eval@local -c user.name=eval commit -qm base --allow-empty ) >/dev/null 2>&1
+  bare="$TMP_DIR/hook-remote.git"; git init --bare -q "$bare"
+  ( cd "$hookrepo" && git remote add origin "$bare" && git push -q origin HEAD:refs/heads/main ) >"$TMP_DIR/gitpush.out" 2>&1
+  if grep -qF PREPUSH-RAN "$TMP_DIR/gitpush.out"; then
+    pass "git dispatches the pre-push hook via core.hooksPath (real git push fires it)"
+  else
+    fail "git push did not fire the pre-push hook via core.hooksPath"; sed -n '1,12p' "$TMP_DIR/gitpush.out"
+  fi
+  # No-fork: hook assets live under assets/, and invoke the veritas CLI — they must not vendor
+  # rule/claim evaluation. (assets/ is outside the adapter/flows no-fork scope; assert explicitly.)
+  if rg -q -i 'evaluateRepoStandards|evidence-check-runner\.mjs|class +[A-Za-z]*RuleEngine' "$KIT/assets/starter-hooks" 2>/dev/null; then
+    fail "governance hook assets appear to vendor Veritas evaluation logic (no-fork violated)"
+  else
+    pass "no-fork: governance hooks invoke the veritas CLI only (no vendored rule/claim engine)"
+  fi
+fi
+
+# --- consult-standards skill (Slice 4 / #649): JIT guidance via `veritas explain` ---------
+# A pull-based advisory skill telling an agent to run `veritas explain --file <path>` before
+# editing. No MCP server; wraps the veritas CLI, reimplements no evaluation.
+echo "--- consult-standards skill ---"
+CONSULT_SKILL="$KIT/skills/consult-standards/SKILL.md"
+if [[ -f "$CONSULT_SKILL" ]]; then
+  pass "consult-standards SKILL.md is present"
+else
+  fail "consult-standards SKILL.md is missing"
+fi
+if rg -q '"id": *"veritas-governance\.consult-standards"' "$KIT/kit.json" && rg -q '"path": *"skills/consult-standards/SKILL.md"' "$KIT/kit.json"; then
+  pass "consult-standards skill is registered in kit.json skills[]"
+else
+  fail "consult-standards skill is NOT registered in kit.json skills[]"
+fi
+# It must point agents at `veritas explain` (the JIT producer), not vendor evaluation.
+if grep -qF 'veritas explain' "$CONSULT_SKILL"; then
+  pass "consult-standards points at the veritas explain CLI"
+else
+  fail "consult-standards does not reference veritas explain"
+fi
+if rg -q -i 'evaluateRepoStandards|evidence-check-runner\.mjs|class +[A-Za-z]*RuleEngine' "$KIT/skills/consult-standards" 2>/dev/null; then
+  fail "consult-standards appears to vendor Veritas evaluation logic (no-fork violated)"
+else
+  pass "no-fork: consult-standards wraps the veritas CLI, no vendored rule/claim engine"
+fi
+# No-MCP invariant (#649 / ADR 0011): the kit ships no MCP server CODE and declares no MCP
+# dependency. Scoped to non-markdown files so the docs' explicit "no MCP server" prose (which
+# names the concept precisely to say the kit avoids it) does not trip the check.
+if rg -q -i 'modelcontextprotocol|new +McpServer|StdioServerTransport|createMcpServer' "$KIT" --glob '!*.md' 2>/dev/null; then
+  fail "kit ships MCP server code/dependency — #649 guidance is a skill, not an MCP server (ADR 0011)"
+else
+  pass "no MCP server: JIT guidance is an agent-pull skill (ADR 0011)"
+fi
+# Live leg: with a Veritas binary, a scaffolded repo's `veritas explain --file` produces JIT context.
+VBIN3="${VERITAS_BIN:-veritas}"
+if command -v "$VBIN3" >/dev/null 2>&1 || [[ -x "$VBIN3" ]]; then
+  cs="$TMP_DIR/consult-repo"; mkdir -p "$cs"
+  ( cd "$cs" && git init -q && printf '{"name":"consult-eval","scripts":{"test":"node -e \\"process.exit(0)\\""}}\n' > package.json && "$VBIN3" init >/dev/null 2>&1 )
+  explain_out="$( cd "$cs" && "$VBIN3" explain --file package.json 2>&1 )"
+  if printf '%s' "$explain_out" | grep -qF 'Veritas JIT Context'; then
+    pass "veritas explain --file produces JIT context on a governed repo (the guidance the skill invokes)"
+  else
+    fail "veritas explain --file did not produce JIT context"; printf '%s\n' "$explain_out" | sed -n '1,8p'
+  fi
+else
+  echo "  - SKIP consult-standards live leg (no Veritas binary; set VERITAS_BIN to enable)"
+fi
+
 echo ""
 if [[ "$errors" -eq 0 ]]; then
   echo "PASS: veritas-governance kit gate demo (positive passes, negative blocks)"