@manhquy/flow-skill 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +5 -5
  3. package/README_VN.md +5 -5
  4. package/package.json +1 -1
  5. package/skills/flow/SKILL.md +108 -219
  6. package/skills/flow/_templates/01-research.md +3 -2
  7. package/skills/flow/_templates/02-scope.md +2 -0
  8. package/skills/flow/_templates/03-prd.md +2 -1
  9. package/skills/flow/_templates/05-contract.md +1 -0
  10. package/skills/flow/_templates/card.md +5 -1
  11. package/skills/flow/law/CLAUDE.md +6 -5
  12. package/skills/flow/law/CODING.md +37 -0
  13. package/skills/flow/references/agent-detection.md +12 -9
  14. package/skills/flow/references/agent-stage-mapping.md +47 -12
  15. package/skills/flow/references/auto-run.md +38 -7
  16. package/skills/flow/references/clarify.md +7 -7
  17. package/skills/flow/references/command-dispatch.md +51 -51
  18. package/skills/flow/references/concierge.md +2 -2
  19. package/skills/flow/references/debt-and-halts.md +20 -5
  20. package/skills/flow/references/gate-00.md +10 -0
  21. package/skills/flow/references/gate-01.md +18 -0
  22. package/skills/flow/references/gate-02.md +16 -0
  23. package/skills/flow/references/gate-03.md +10 -0
  24. package/skills/flow/references/gate-04.md +12 -0
  25. package/skills/flow/references/gate-05.md +33 -0
  26. package/skills/flow/references/gate-assess.md +13 -0
  27. package/skills/flow/references/gate-card.md +25 -0
  28. package/skills/flow/references/gate-consistency.md +24 -0
  29. package/skills/flow/references/gate-constitution.md +19 -0
  30. package/skills/flow/references/gate-debt.md +5 -0
  31. package/skills/flow/references/gate-eval.md +54 -3
  32. package/skills/flow/references/gate-examples.md +78 -0
  33. package/skills/flow/references/gate-rules.md +40 -241
  34. package/skills/flow/references/gate-shared.md +37 -0
  35. package/skills/flow/references/ground-truth-gates.md +1 -1
  36. package/skills/flow/references/host-agnostic-parallel.md +54 -0
  37. package/skills/flow/references/mode-work.md +13 -10
  38. package/skills/flow/runner/flow.sh +334 -58
  39. package/skills-manifest.json +17 -2
@@ -171,6 +171,7 @@ current_stage_idx() { # highest CONTIGUOUS-from-00 stage index, or -1 if 00 miss
171
171
  # Prints violations; returns 0 = clean, 1 = violations found.
172
172
  scan_gate() {
173
173
  local file="$1" found=0 unchecked fills
174
+ _MIDTIER_FAILS=0
174
175
  if [ ! -f "$file" ]; then
175
176
  echo " missing file: $file"
176
177
  return 1
@@ -187,9 +188,132 @@ scan_gate() {
187
188
  printf '%s\n' "$fills" | sed 's/^/ L/'
188
189
  found=1
189
190
  fi
191
+ case "$(basename "$file")" in
192
+ 01-research.md|02-scope.md|03-prd.md|05-contract.md)
193
+ if ! _scan_midtier "$file"; then
194
+ found=1
195
+ fi
196
+ ;;
197
+ esac
190
198
  return $found
191
199
  }
192
200
 
201
+ # Materialize a ## heading body into stdout (one awk; caller assigns to a var).
202
+ # Copy of cmd_clarify's heading walk: start at heading, stop at the next ##.
203
+ _heading_slice() { # $1=file $2=heading prefix after "## "
204
+ local file="$1" heading="$2"
205
+ awk -v h="$heading" '
206
+ {
207
+ line = $0
208
+ sub(/\r$/, "", line)
209
+ }
210
+ line ~ ("^##[[:space:]]+" h) { f=1; next }
211
+ /^##[[:space:]]/ { f=0 }
212
+ f { print line }
213
+ ' "$file"
214
+ }
215
+
216
+ _midtier_hit() {
217
+ echo " [x] $1"
218
+ _MIDTIER_FAILS=$((_MIDTIER_FAILS + 1))
219
+ }
220
+
221
+ _midtier_entry_n() { # stdin = ## What exists already -> numbered or bullet entries
222
+ awk '
223
+ /^[[:space:]]*[0-9]+\./ { c++; next }
224
+ /^[[:space:]]*- / { c++ }
225
+ END { print c + 0 }
226
+ '
227
+ }
228
+
229
+ _midtier_l_above_a() { # stdin = ## Features in v1 body; exit 1 if a joined bullet is L + B/C
230
+ awk '
231
+ function flush() {
232
+ if (buf == "") return
233
+ low = tolower(buf)
234
+ hasL = (low ~ /impact[[:space:]]*:?[[:space:]]*l([^a-z]|$)/ || low ~ /(^|[^a-z])l-impact/)
235
+ hasBC = (low ~ /grade[[:space:]]*:?[[:space:]]*[bc]([^a-z]|$)/)
236
+ if (hasL && hasBC) bad++
237
+ buf = ""
238
+ }
239
+ /^[[:space:]]*- / { flush(); buf = $0; next }
240
+ /^[[:space:]]*$/ { flush(); next }
241
+ buf != "" { buf = buf " " $0; next }
242
+ { flush() }
243
+ END { flush(); exit (bad > 0) ? 1 : 0 }
244
+ '
245
+ }
246
+
247
+ _midtier_adjectives() { # stdin = Features or NFR body; print first listed adjective
248
+ awk '
249
+ /^[[:space:]]*- \[[ xX]\]/ { next }
250
+ {
251
+ line = tolower($0)
252
+ n = split("fast secure intuitive robust prominent", w, " ")
253
+ for (i = 1; i <= n; i++) {
254
+ if (line ~ "(^|[^a-z])" w[i] "([^a-z]|$)") { print w[i]; exit }
255
+ }
256
+ }
257
+ '
258
+ }
259
+
260
+ _midtier_od_n() { # stdin = ## Open decisions body -> markdown bullet count (checked or not)
261
+ awk '/^[[:space:]]*- / { c++ } END { print c + 0 }'
262
+ }
263
+
264
+ _midtier_open_decisions() { # $1=file
265
+ local body n
266
+ body="$(_heading_slice "$1" "Open decisions")"
267
+ n="$(printf '%s\n' "$body" | _midtier_od_n)"
268
+ if [ "$n" -gt 5 ]; then
269
+ _midtier_hit "more than 5 ## Open decisions bullets ($n)"
270
+ fi
271
+ }
272
+
273
+ _scan_midtier() { # $1=file; prints [x] lines; return 1 if any hit
274
+ local file="$1" base body n pt adj
275
+ base="$(basename "$file")"
276
+ case "$base" in
277
+ 01-research.md)
278
+ if [ -f "$PROJECT_TYPE_FILE" ]; then
279
+ pt="$(tr -d '\r' < "$PROJECT_TYPE_FILE" | awk 'NF{print; exit}')"
280
+ if [ "$pt" = "web" ]; then
281
+ body="$(_heading_slice "$file" "What exists already")"
282
+ n="$(printf '%s\n' "$body" | _midtier_entry_n)"
283
+ if [ "$n" -lt 3 ]; then
284
+ _midtier_hit "fewer than 3 entries under ## What exists already (typed web, got $n)"
285
+ fi
286
+ fi
287
+ fi
288
+ ;;
289
+ 02-scope.md)
290
+ body="$(_heading_slice "$file" "Features in v1")"
291
+ if ! printf '%s\n' "$body" | _midtier_l_above_a; then
292
+ _midtier_hit "L-impact feature above grade A in ## Features in v1"
293
+ fi
294
+ _midtier_open_decisions "$file"
295
+ ;;
296
+ 03-prd.md)
297
+ body="$(_heading_slice "$file" "Features")"
298
+ adj="$(printf '%s\n' "$body" | _midtier_adjectives)"
299
+ if [ -n "$adj" ]; then
300
+ _midtier_hit "unquantified adjective '$adj' in ## Features"
301
+ else
302
+ body="$(_heading_slice "$file" "Non-functional requirements")"
303
+ adj="$(printf '%s\n' "$body" | _midtier_adjectives)"
304
+ if [ -n "$adj" ]; then
305
+ _midtier_hit "unquantified adjective '$adj' in ## Non-functional requirements"
306
+ fi
307
+ fi
308
+ _midtier_open_decisions "$file"
309
+ ;;
310
+ 05-contract.md)
311
+ _midtier_open_decisions "$file"
312
+ ;;
313
+ esac
314
+ [ "$_MIDTIER_FAILS" -eq 0 ]
315
+ }
316
+
193
317
  _python() {
194
318
  # prefer python3, and only accept an interpreter that is actually Python 3.x (the harness +
195
319
  # repo_map.py are py3-only — a py2 `python` must NOT be selected or it fails silently).
@@ -437,6 +561,18 @@ planning_complete() { # 0 = yes: each stage is clean, OR debt-skipped (a skipped
437
561
  return 0
438
562
  }
439
563
 
564
+ # Files exist (or debt-skipped). Used by project-type lock so a midtier-dirty
565
+ # 01 cannot unlock a type flip after stages 00-05 are on disk.
566
+ planning_artifacts_present() {
567
+ local s
568
+ for s in $STAGES; do
569
+ if [ ! -f "$FLOW_DIR/$s.md" ]; then
570
+ stage_skipped "$s" || return 1
571
+ fi
572
+ done
573
+ return 0
574
+ }
575
+
440
576
  # project type (web|cli|library|skill); absent => web. Adapts done-evidence + guidance.
441
577
  get_project_type() {
442
578
  local t; t="$(cat "$PROJECT_TYPE_FILE" 2>/dev/null | tr -d '\r' | awk 'NF{print; exit}')"
@@ -759,7 +895,7 @@ cmd_status() {
759
895
  echo " auto: $_ast$([ "$_ast" = ACTIVE ] && echo " (receipts enforced)" || true)"
760
896
  lock_warn
761
897
  echo
762
- echo "NEXT -> $(_next_action)"
898
+ _emit_next
763
899
  echo
764
900
  if [ -f "$FLOW_DIR/00-inspect.md" ]; then
765
901
  if scan_gate "$FLOW_DIR/00-inspect.md" >/dev/null 2>&1; then
@@ -856,7 +992,9 @@ _resume_valid_lines() {
856
992
  cmd_resume() {
857
993
  local idx; idx="$(current_stage_idx)"
858
994
  if [ "$idx" -lt 0 ]; then
859
- echo "nothing to resume - $(_next_action)"
995
+ _na="$(_next_action)"
996
+ echo "nothing to resume - $_na"
997
+ echo "NEXT_VERB=$(_next_verb_from "$_na")"
860
998
  return 0
861
999
  fi
862
1000
 
@@ -874,7 +1012,7 @@ cmd_resume() {
874
1012
  echo
875
1013
  _gate_state_brief "$idx"
876
1014
  echo
877
- echo "NEXT -> $(_next_action)"
1015
+ _emit_next
878
1016
  return 0
879
1017
  fi
880
1018
 
@@ -966,7 +1104,7 @@ cmd_resume() {
966
1104
  echo
967
1105
 
968
1106
  # ---- NEXT (shared with `status` via _next_action - the two verbs can never disagree) ----
969
- echo "NEXT -> $(_next_action)"
1107
+ _emit_next
970
1108
  }
971
1109
 
972
1110
  cmd_next() {
@@ -1005,7 +1143,7 @@ cmd_next() {
1005
1143
  # attribute the failing event to THIS stage (so usage top-fail-stage + propose can see it)
1006
1144
  # and record WHICH checks failed, so a chronically-failing stage is diagnosable.
1007
1145
  FLOW_LOG_STAGE_TO="$cur"
1008
- FLOW_LAST_GATE_FAIL="fill:$(grep -c '\[FILL' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0),unchecked:$(grep -cE '^[[:space:]]*- \[ \]' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0)"
1146
+ FLOW_LAST_GATE_FAIL="fill:$(grep -c '\[FILL' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0),unchecked:$(grep -cE '^[[:space:]]*- \[ \]' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0),midtier:${_MIDTIER_FAILS:-0}"
1009
1147
  echo
1010
1148
  echo "Fix the above, then run '/flow next' again. (Kill at a gate is also valid.)"
1011
1149
  return 1
@@ -1179,6 +1317,27 @@ _next_action() {
1179
1317
  echo "run '/flow check' then ship per stage 09"
1180
1318
  }
1181
1319
 
1320
+ # Closed advisory enum from _next_action prose only. Hosts must not auto-exec auto/skip.
1321
+ _next_verb_from() { # $1 = _next_action text
1322
+ local a="$1"
1323
+ case "$a" in
1324
+ *"fix gate:"*) echo fix-gate ;;
1325
+ *"/flow card start"*) echo card-start ;;
1326
+ *"/flow card"*) echo card ;;
1327
+ *"/flow check"*) echo check ;;
1328
+ *"/flow next"*) echo next ;;
1329
+ *"in flight"*) echo card-start ;;
1330
+ *) echo none ;;
1331
+ esac
1332
+ }
1333
+
1334
+ _emit_next() {
1335
+ local a
1336
+ a="$(_next_action)"
1337
+ echo "NEXT -> $a"
1338
+ echo "NEXT_VERB=$(_next_verb_from "$a")"
1339
+ }
1340
+
1182
1341
  _set_card_status() { # $1=file $2=value -> rewrite the ^status: line (portable substitute; temp+mv)
1183
1342
  local f="$1" v="$2"
1184
1343
  sed "s/^status:.*/status: $v/" "$f" > "$f.tmp" 2>/dev/null || { rm -f "$f.tmp"; return 1; }
@@ -1590,12 +1749,13 @@ cmd_project_type() {
1590
1749
  fi
1591
1750
  case "$arg" in
1592
1751
  web|cli|library|skill)
1593
- # After planning is complete AND an explicit type file exists, refuse silent
1752
+ # After planning artifacts exist AND an explicit type file exists, refuse silent
1594
1753
  # type flips unless FLOW_FORCE=1 (D6). Default-web with no file is not "locked".
1595
- if planning_complete 2>/dev/null && [ -f "$PROJECT_TYPE_FILE" ]; then
1754
+ # Gate cleanliness is NOT required: a midtier-dirty 01 must not unlock a flip.
1755
+ if planning_artifacts_present 2>/dev/null && [ -f "$PROJECT_TYPE_FILE" ]; then
1596
1756
  local cur; cur="$(get_project_type)"
1597
1757
  if [ -n "$cur" ] && [ "$cur" != "$arg" ] && [ "${FLOW_FORCE:-0}" != "1" ]; then
1598
- echo "FAIL: project type is locked to '$cur' after planning completes (would change done-evidence lens)."
1758
+ echo "FAIL: project type is locked to '$cur' once planning artifacts exist (would change done-evidence lens)."
1599
1759
  echo " re-set with FLOW_FORCE=1 if intentional, and record a DEBT.md line for the change."
1600
1760
  return 1
1601
1761
  fi
@@ -2302,6 +2462,14 @@ cmd_doctor() {
2302
2462
  command -v cargo >/dev/null 2>&1 && echo " cargo: ok (optional Rust harness power-path)" || true
2303
2463
  [ -d "$TEMPLATE_DIR" ] && echo " templates: ok" || { echo " templates: MISSING ($TEMPLATE_DIR)"; ok=0; }
2304
2464
  [ -f "$HARNESS_PY" ] && echo " harness: present ($HARNESS_PY)" || echo " harness: absent (durable layer off)"
2465
+ if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then
2466
+ echo " timeout: full (timeout|gtimeout present)"
2467
+ else
2468
+ case "$(uname -s 2>/dev/null)" in
2469
+ Darwin) echo " timeout: absent (no timeout/gtimeout; live eval REFUSED)" ;;
2470
+ *) echo " timeout: partial (watchdog fallback; live eval not GNU-bounded)" ;;
2471
+ esac
2472
+ fi
2305
2473
  echo
2306
2474
  if [ "$ok" -eq 1 ]; then
2307
2475
  echo "READY. Gate engine works$([ -n "$py" ] && echo '; durable layer on.' || echo '; durable layer off (no python).')"
@@ -3154,17 +3322,26 @@ _eval_heading_pattern() {
3154
3322
  esac
3155
3323
  }
3156
3324
 
3157
- # Extract one stage's ritual text: from its heading line up to (not including) the next '## '
3158
- # heading, or EOF. Prints nothing on a bad stage/missing file - caller MUST assert non-empty
3159
- # before any billable call (a silent empty extraction would judge against no rule at all).
3325
+ # Extract one stage's ritual text from the mapped file (01→gate-01.md, 02→gate-02.md,
3326
+ # card→gate-card.md): from its heading line up to (not including) the next '## '
3327
+ # heading, or EOF. Prints nothing on a bad stage/missing file - caller MUST assert
3328
+ # non-empty before any billable call (a silent empty extraction would judge against
3329
+ # no rule at all).
3160
3330
  _eval_extract_section() { # $1 = stage (01|02|card)
3161
- local pat; pat="$(_eval_heading_pattern "$1")" || return 1
3162
- [ -f "$GATE_RULES_FILE" ] || return 1
3331
+ local pat src
3332
+ pat="$(_eval_heading_pattern "$1")" || return 1
3333
+ case "$1" in
3334
+ 01) src="$SCRIPT_DIR/../references/gate-01.md" ;;
3335
+ 02) src="$SCRIPT_DIR/../references/gate-02.md" ;;
3336
+ card) src="$SCRIPT_DIR/../references/gate-card.md" ;;
3337
+ *) return 1 ;;
3338
+ esac
3339
+ [ -f "$src" ] || return 1
3163
3340
  awk -v pat="$pat" '
3164
3341
  $0 ~ pat { f=1; print; next }
3165
3342
  f && /^## / { f=0 }
3166
3343
  f { print }
3167
- ' "$GATE_RULES_FILE"
3344
+ ' "$src"
3168
3345
  }
3169
3346
 
3170
3347
  # Usability probe: zero cost if `claude` is absent; exactly one minimal billable call if present.
@@ -3199,13 +3376,39 @@ _eval_engine_run() { # $1=promptfile $2=timeout-seconds -> stdout=raw json; retu
3199
3376
  # command string above. Empirical: adding a `2> '$errfile'` inside the timed cmd breaks the
3200
3377
  # watchdog-fallback path in `_run_with_timeout` on the timeout-less-PATH lane (macOS DEBT lane).
3201
3378
  # Outer-scope redirection is exactly the same capture at zero cost to the timeout contract.
3379
+ # Opt-in isolated cwd for live eval (Approach D). Unset = today's cwd (byte-identical
3380
+ # call path). Set but not a directory = fail loud. Isolation is cd+env around existing
3381
+ # probe/engine calls — never argv, never a second engine. CLAUDE_CODE_SAFE_MODE=1 is
3382
+ # exported ONLY in the isolated subshell (official Claude env; keeps OAuth; blast radius
3383
+ # is the judge child). Do not invent a flow alias; do not set HOME or CLAUDE_CONFIG_DIR.
3384
+ _eval_isolated_guard() {
3385
+ [ -z "${FLOW_EVAL_ISOLATED_CWD:-}" ] && return 0
3386
+ [ -d "$FLOW_EVAL_ISOLATED_CWD" ] && return 0
3387
+ echo "FAIL: FLOW_EVAL_ISOLATED_CWD is set but is not a directory: $FLOW_EVAL_ISOLATED_CWD"
3388
+ return 1
3389
+ }
3390
+ _eval_isolated_run() {
3391
+ if [ -n "${FLOW_EVAL_ISOLATED_CWD:-}" ]; then
3392
+ ( cd "$FLOW_EVAL_ISOLATED_CWD" && export CLAUDE_CODE_SAFE_MODE=1 && "$@" )
3393
+ else
3394
+ "$@"
3395
+ fi
3396
+ }
3397
+
3202
3398
 
3203
- # Build the judge prompt for one fixture. Returns 1 (writes nothing usable) if the gate-rules.md
3204
- # section extraction came back empty.
3399
+ # Build the judge prompt for one fixture. The challenge block is the extract output
3400
+ # (already the judge bytes; `_eval_gate_rules_sha` is not stdin). Returns 1 if empty.
3205
3401
  _eval_build_prompt() { # $1=outfile $2=stage $3=artifact-file $4=nonce
3206
- local outfile="$1" stage="$2" artifact="$3" nonce="$4" section
3402
+ local outfile="$1" stage="$2" artifact="$3" nonce="$4" section shared
3207
3403
  section="$(_eval_extract_section "$stage")" || return 1
3208
3404
  [ -n "$section" ] || return 1
3405
+ # Shared OD/authority lives in 02/03/05 challenges. Extract maps 01/02/card only;
3406
+ # appending shared onto 01/card lets OD rules false-FLAG research/evidence.
3407
+ if [ "$stage" = "02" ]; then
3408
+ shared="$SCRIPT_DIR/../references/gate-shared.md"
3409
+ [ -f "$shared" ] && section="$section
3410
+ $(cat "$shared")"
3411
+ fi
3209
3412
  {
3210
3413
  printf 'You are reviewing a build-process artifact against the quality-gate challenge below.\n'
3211
3414
  printf 'Read the challenge, then the artifact, then decide honestly: does the artifact\n'
@@ -3284,11 +3487,34 @@ _eval_nonce_epoch() { # $1=run_id -> epoch seconds (or empty)
3284
3487
  printf '%s' "$1" | awk -F- 'NF>=2 { print $(NF-1) }'
3285
3488
  }
3286
3489
 
3287
- # CRLF-normalized gate-rules.md hash: a raw-byte hash would differ per checkout line-endings
3288
- # (git autocrlf) and produce a false "prose changed" drift alarm across identical content.
3490
+ # CRLF-normalized hash of the explicit gate-rules corpus (index + shared + 11
3491
+ # section files). Never glob `gate-*.md` (would pull gate-eval.md + gate-examples.md).
3492
+ # A raw-byte hash would differ per checkout line-endings (git autocrlf).
3289
3493
  _eval_gate_rules_sha() {
3290
- [ -f "$GATE_RULES_FILE" ] || { printf 'unknown'; return; }
3291
- tr -d '\r' < "$GATE_RULES_FILE" | cksum | awk '{print $1}'
3494
+ local ref="$SCRIPT_DIR/../references"
3495
+ [ -f "$ref/gate-rules.md" ] || { printf 'unknown'; return; }
3496
+ {
3497
+ cat "$ref/gate-rules.md"
3498
+ [ -f "$ref/gate-shared.md" ] && cat "$ref/gate-shared.md"
3499
+ [ -f "$ref/gate-assess.md" ] && cat "$ref/gate-assess.md"
3500
+ [ -f "$ref/gate-00.md" ] && cat "$ref/gate-00.md"
3501
+ [ -f "$ref/gate-01.md" ] && cat "$ref/gate-01.md"
3502
+ [ -f "$ref/gate-02.md" ] && cat "$ref/gate-02.md"
3503
+ [ -f "$ref/gate-03.md" ] && cat "$ref/gate-03.md"
3504
+ [ -f "$ref/gate-04.md" ] && cat "$ref/gate-04.md"
3505
+ [ -f "$ref/gate-05.md" ] && cat "$ref/gate-05.md"
3506
+ [ -f "$ref/gate-card.md" ] && cat "$ref/gate-card.md"
3507
+ [ -f "$ref/gate-consistency.md" ] && cat "$ref/gate-consistency.md"
3508
+ [ -f "$ref/gate-constitution.md" ] && cat "$ref/gate-constitution.md"
3509
+ [ -f "$ref/gate-debt.md" ] && cat "$ref/gate-debt.md"
3510
+ } | tr -d '\r' | cksum | awk '{print $1}'
3511
+ }
3512
+
3513
+ # CRLF-normalized assembler-output hash (same recipe as _eval_gate_rules_sha). Covers
3514
+ # prompt-file bytes only — not host CLAUDE.md. $1=prompt file.
3515
+ _eval_prompt_sha() {
3516
+ [ -f "$1" ] || { printf 'unknown'; return; }
3517
+ tr -d '\r' < "$1" | cksum | awk '{print $1}'
3292
3518
  }
3293
3519
 
3294
3520
  # `claude --version` is the CLI version (e.g. "2.1.201 (Claude Code)"), NOT a model id - kept
@@ -3317,9 +3543,10 @@ _eval_replay_meta_get() { # $1=key -> value (empty + return 1 if missing)
3317
3543
  return 1
3318
3544
  }
3319
3545
 
3320
- _eval_replay_write_meta() { # $1=nonce $2=gate_rules_sha $3=model $4=n
3546
+ _eval_replay_write_meta() { # $1=nonce $2=gate_rules_sha $3=model $4=n [$5=prompt_sha.<fid>= lines]
3321
3547
  mkdir -p "$EVAL_REPLAY_DIR" || return 1
3322
- printf 'nonce=%s\ngate_rules_sha=%s\nmodel=%s\nn=%s\n' "$1" "$2" "$3" "$4" > "$EVAL_REPLAY_DIR/meta"
3548
+ printf 'nonce=%s\ngate_rules_sha=%s\nmodel=%s\nn=%s\n' "$1" "$2" "$3" "$4" > "$EVAL_REPLAY_DIR/meta" || return 1
3549
+ [ -n "${5:-}" ] && printf '%s' "$5" >> "$EVAL_REPLAY_DIR/meta"
3323
3550
  }
3324
3551
 
3325
3552
  # Persist one stripped verdict line only — never a JSON envelope (session_id/cwd stay out).
@@ -3366,17 +3593,18 @@ _eval_emit_batch_marker() { # $1=run_id $2=start|done $3=n_expected-or-n_written
3366
3593
  # cli_version/flow_version/gate_rules_sha are passed in (computed ONCE per batch by the caller,
3367
3594
  # not per fixture row) - they never change mid-batch, and recomputing per-row was both wasteful
3368
3595
  # (an extra subprocess per fixture) and the original vector for the stdin-consumption bug above.
3369
- _eval_emit_result() { # $1..$14 as before + $15=retries $16=rate_limited (true|false)
3596
+ _eval_emit_result() { # $1..$14 as before + $15=retries $16=rate_limited $17=prompt_sha $18=timed_out
3370
3597
  local run_id="$1" fid="$2" stage="$3" expected="$4" verdict="$5" match="$6" flag="$7" pass="$8" invalid="$9" n="${10}" model="${11}" cliv="${12}" flowv="${13}" grsha="${14}"
3371
- local retries="${15:-0}" ratel="${16:-false}"
3598
+ local retries="${15:-0}" ratel="${16:-false}" psha="${17:-}" timedout="${18:-false}"
3372
3599
  case "$retries" in ''|*[!0-9]*) retries=0 ;; esac
3373
3600
  case "$ratel" in true|false) : ;; *) ratel=false ;; esac
3601
+ case "$timedout" in true|false) : ;; *) timedout=false ;; esac
3374
3602
  mkdir -p "$LOG_DIR" 2>/dev/null || true
3375
- printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","stage":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"flag":%s,"pass":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","gate_rules_sha":"%s","retries":%s,"rate_limited":%s}\n' \
3603
+ printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","stage":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"flag":%s,"pass":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","gate_rules_sha":"%s","retries":%s,"rate_limited":%s,"prompt_sha":"%s","timed_out":%s}\n' \
3376
3604
  "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" "$(_now)" "$(_json_str "$run_id")" "$(_json_str "$fid")" \
3377
3605
  "$(_json_str "$stage")" "$(_json_str "$expected")" "$(_json_str "$verdict")" "$(_json_str "$match")" \
3378
3606
  "$flag" "$pass" "$invalid" "$n" "$(_json_str "$cliv")" "$(_json_str "$model")" \
3379
- "$(_json_str "$flowv")" "$(_json_str "$grsha")" "$retries" "$ratel" >> "$EVAL_RESULTS_FILE" 2>/dev/null || true
3607
+ "$(_json_str "$flowv")" "$(_json_str "$grsha")" "$retries" "$ratel" "$(_json_str "$psha")" "$timedout" >> "$EVAL_RESULTS_FILE" 2>/dev/null || true
3380
3608
  }
3381
3609
 
3382
3610
  # Strip the noisy `system`/`init` envelope from a raw --output-format json line before persisting
@@ -3660,17 +3888,18 @@ _eval_routing_parse_action() {
3660
3888
  [ -n "$action" ] && printf '%s' "$action" || echo INVALID
3661
3889
  }
3662
3890
 
3663
- _eval_routing_emit_result() { # $1=run_id $2=fid $3=expected $4=verdict $5=match $6..$9=votes-hit/miss/invalid/n $10=model $11=cliv $12=flowv $13=rulessha $14=retries $15=rate_limited
3891
+ _eval_routing_emit_result() { # $1=run_id $2=fid $3=expected $4=verdict $5=match $6..$9=votes $10=model $11=cliv $12=flowv $13=rulessha $14=retries $15=rate_limited $16=prompt_sha $17=timed_out
3664
3892
  local run_id="$1" fid="$2" expected="$3" verdict="$4" match="$5" hit="$6" miss="$7" invalid="$8" n="$9"
3665
- local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}"
3893
+ local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}" psha="${16:-}" timedout="${17:-false}"
3666
3894
  case "$retries" in ''|*[!0-9]*) retries=0 ;; esac
3667
3895
  case "$ratel" in true|false) : ;; *) ratel=false ;; esac
3896
+ case "$timedout" in true|false) : ;; *) timedout=false ;; esac
3668
3897
  mkdir -p "$LOG_DIR" 2>/dev/null || true
3669
- printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s}\n' \
3898
+ printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s,"prompt_sha":"%s","timed_out":%s}\n' \
3670
3899
  "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" "$(_now)" "$(_json_str "$run_id")" "$(_json_str "$fid")" \
3671
3900
  "$(_json_str "$expected")" "$(_json_str "$verdict")" "$(_json_str "$match")" \
3672
3901
  "$hit" "$miss" "$invalid" "$n" "$(_json_str "$cliv")" "$(_json_str "$model")" \
3673
- "$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" >> "$EVAL_ROUTING_RESULTS_FILE" 2>/dev/null || true
3902
+ "$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" "$(_json_str "$psha")" "$timedout" >> "$EVAL_ROUTING_RESULTS_FILE" 2>/dev/null || true
3674
3903
  }
3675
3904
 
3676
3905
  _eval_routing_complete_run_ids() {
@@ -3744,6 +3973,9 @@ _eval_routing_print_scorecard() { # $1=run_id
3744
3973
  '
3745
3974
  }
3746
3975
 
3976
+ _eval_skip_unmeasured() { # $1 = kind (eval / routing eval / converge eval)
3977
+ echo "SKIP: 'claude' CLI not found on PATH - $1 needs it to run the LLM judge. Zero calls made."
3978
+ }
3747
3979
  # NOTE: defined with the `function name { }` form (no parens) rather than this file's usual
3748
3980
  # `name() { }` style, solely so the literal 4-byte substring the shipped verb name is built
3749
3981
  # from never appears immediately followed by '(' in the source - a blind text-pattern security
@@ -3770,10 +4002,13 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
3770
4002
  return 1
3771
4003
  fi
3772
4004
 
3773
- local probe; probe="$(_eval_probe)"
4005
+ _eval_isolated_guard || return 1
4006
+
4007
+ local probe; probe="$(_eval_isolated_run _eval_probe)"
3774
4008
  case "$probe" in
3775
4009
  absent)
3776
- echo "SKIP: 'claude' CLI not found on PATH - routing eval needs it to run the LLM judge. Zero calls made."
4010
+ _eval_skip_unmeasured "routing eval"
4011
+ echo "semantic layer unmeasured on this host"
3777
4012
  return 0 ;;
3778
4013
  fail)
3779
4014
  echo "SKIP: 'claude' CLI is present but the sentinel probe did not come back clean (one minimal"
@@ -3838,14 +4073,18 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
3838
4073
  continue
3839
4074
  fi
3840
4075
 
4076
+ local psha
4077
+ psha="$(_eval_prompt_sha "$promptfile")"
4078
+
3841
4079
  total_evaluated=$((total_evaluated + 1))
3842
- local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false
4080
+ local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false vote_timed_out=false
3843
4081
  while [ "$i" -le "$n" ]; do
3844
4082
  local raw rc v raw1="" rc1=0
3845
- raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
4083
+ raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
3846
4084
  raw1="$raw"; rc1="$rc"
3847
4085
  if [ "$rc" -eq 124 ]; then
3848
4086
  v=INVALID
4087
+ vote_timed_out=true
3849
4088
  else
3850
4089
  v="$(_eval_routing_parse_action "$raw" "$nonce")"
3851
4090
  [ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"
@@ -3857,9 +4096,9 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
3857
4096
  echo " $fid: retrying vote $i after ${backoff}s (parse-INVALID on attempt 1)"
3858
4097
  sleep "$backoff" 2>/dev/null || true
3859
4098
  fi
3860
- raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
4099
+ raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
3861
4100
  retries_sum=$((retries_sum + 1))
3862
- if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_routing_parse_action "$raw" "$nonce")"; fi
4101
+ if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_routing_parse_action "$raw" "$nonce")"; fi
3863
4102
  [ "$model" = "unknown" ] && [ "$rc" -ne 124 ] && model="$(_eval_parse_model "$raw")"
3864
4103
  local rl2; rl2="$(_eval_parse_rate_limited "$raw")"
3865
4104
  [ "$rl2" = "true" ] && vote_rate_limited=true
@@ -3897,7 +4136,7 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
3897
4136
  echo " $fid: MISS - expected '$fexpected' (hit=$hit_count miss=$miss_count invalid=$invalid_count)"
3898
4137
  fi
3899
4138
 
3900
- _eval_routing_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited"
4139
+ _eval_routing_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited" "$psha" "$vote_timed_out"
3901
4140
  n_written=$((n_written + 1))
3902
4141
 
3903
4142
  if [ "$total_evaluated" -eq 1 ] && [ "$verdict" = "UNRELIABLE" ] && [ "$keep_going" -eq 0 ]; then
@@ -4005,15 +4244,16 @@ _eval_converge_parse_verdict() { # $1=raw $2=nonce -> GAP|CONVERGED|INVALID
4005
4244
 
4006
4245
  _eval_converge_emit_result() { # same field shape as _eval_routing_emit_result
4007
4246
  local run_id="$1" fid="$2" expected="$3" verdict="$4" match="$5" hit="$6" miss="$7" invalid="$8" n="$9"
4008
- local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}"
4247
+ local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}" psha="${16:-}" timedout="${17:-false}"
4009
4248
  case "$retries" in ''|*[!0-9]*) retries=0 ;; esac
4010
4249
  case "$ratel" in true|false) : ;; *) ratel=false ;; esac
4250
+ case "$timedout" in true|false) : ;; *) timedout=false ;; esac
4011
4251
  mkdir -p "$LOG_DIR" 2>/dev/null || true
4012
- printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s}\n' \
4252
+ printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s,"prompt_sha":"%s","timed_out":%s}\n' \
4013
4253
  "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" "$(_now)" "$(_json_str "$run_id")" "$(_json_str "$fid")" \
4014
4254
  "$(_json_str "$expected")" "$(_json_str "$verdict")" "$(_json_str "$match")" \
4015
4255
  "$hit" "$miss" "$invalid" "$n" "$(_json_str "$cliv")" "$(_json_str "$model")" \
4016
- "$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" >> "$EVAL_CONVERGE_RESULTS_FILE" 2>/dev/null || true
4256
+ "$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" "$(_json_str "$psha")" "$timedout" >> "$EVAL_CONVERGE_RESULTS_FILE" 2>/dev/null || true
4017
4257
  }
4018
4258
 
4019
4259
  _eval_converge_emit_batch_marker() { # $1=run_id $2=start|done $3=n
@@ -4075,9 +4315,11 @@ function cmd_eval_converge { # same arg shape as cmd_eval, called only when --st
4075
4315
  return 1
4076
4316
  fi
4077
4317
 
4078
- local probe; probe="$(_eval_probe)"
4318
+ _eval_isolated_guard || return 1
4319
+
4320
+ local probe; probe="$(_eval_isolated_run _eval_probe)"
4079
4321
  case "$probe" in
4080
- absent) echo "SKIP: 'claude' CLI not found on PATH - converge eval needs it to run the LLM judge. Zero calls made."; return 0 ;;
4322
+ absent) _eval_skip_unmeasured "converge eval"; echo "semantic layer unmeasured on this host"; return 0 ;;
4081
4323
  fail) echo "SKIP: 'claude' CLI present but the sentinel probe did not come back clean (one minimal probe call was made). Retry once 'claude -p' runs headless."; return 0 ;;
4082
4324
  esac
4083
4325
 
@@ -4123,17 +4365,20 @@ function cmd_eval_converge { # same arg shape as cmd_eval, called only when --st
4123
4365
  echo " $fid: FAIL - could not build converge prompt (converge.md/repo-dir/challenge missing)"; total_miss=$((total_miss + 1)); rm -rf "$rundir" 2>/dev/null; continue
4124
4366
  fi
4125
4367
 
4368
+ local psha
4369
+ psha="$(_eval_prompt_sha "$promptfile")"
4370
+
4126
4371
  total_evaluated=$((total_evaluated + 1))
4127
- local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false
4372
+ local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false vote_timed_out=false
4128
4373
  while [ "$i" -le "$n" ]; do
4129
4374
  local raw rc v raw1="" rc1=0
4130
- raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; raw1="$raw"; rc1="$rc"
4131
- if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; [ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"; fi
4375
+ raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; raw1="$raw"; rc1="$rc"
4376
+ if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; [ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"; fi
4132
4377
  local rl1; rl1="$(_eval_parse_rate_limited "$raw1")"; [ "$rl1" = "true" ] && vote_rate_limited=true
4133
4378
  if [ "$v" = "INVALID" ] && [ "$rl1" != "true" ] && [ "$rc1" -ne 124 ]; then
4134
4379
  if [ "$backoff" -gt 0 ]; then echo " $fid: retrying vote $i after ${backoff}s (parse-INVALID on attempt 1)"; sleep "$backoff" 2>/dev/null || true; fi
4135
- raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; retries_sum=$((retries_sum + 1))
4136
- if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; fi
4380
+ raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; retries_sum=$((retries_sum + 1))
4381
+ if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; fi
4137
4382
  [ "$model" = "unknown" ] && [ "$rc" -ne 124 ] && model="$(_eval_parse_model "$raw")"
4138
4383
  local rl2; rl2="$(_eval_parse_rate_limited "$raw")"; [ "$rl2" = "true" ] && vote_rate_limited=true
4139
4384
  fi
@@ -4158,7 +4403,7 @@ function cmd_eval_converge { # same arg shape as cmd_eval, called only when --st
4158
4403
  total_miss=$((total_miss + 1)); echo " $fid: MISS - expected '$fexpected' (hit=$hit_count miss=$miss_count invalid=$invalid_count)"
4159
4404
  fi
4160
4405
 
4161
- _eval_converge_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited"
4406
+ _eval_converge_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited" "$psha" "$vote_timed_out"
4162
4407
  n_written=$((n_written + 1))
4163
4408
 
4164
4409
  if [ "$total_evaluated" -eq 1 ] && [ "$verdict" = "UNRELIABLE" ] && [ "$keep_going" -eq 0 ]; then
@@ -4189,6 +4434,8 @@ _eval_guard_unbounded_darwin() { # $1=replay_mode (0/1); return 1 = refused
4189
4434
  local replay_mode="${1:-0}"
4190
4435
  [ "$replay_mode" -eq 0 ] || return 0
4191
4436
  [ "${FLOW_EVAL_UNBOUNDED:-}" = "1" ] && return 0
4437
+ # No judge binary => no stuck call to bound. Probe SKIP (unmeasured) owns this case.
4438
+ command -v claude >/dev/null 2>&1 || return 0
4192
4439
  local darwin=0
4193
4440
  if [ "${FLOW_EVAL_FORCE_DARWIN:-}" = "1" ]; then
4194
4441
  darwin=1
@@ -4297,7 +4544,7 @@ function cmd_eval {
4297
4544
 
4298
4545
  local nonce="" total_mismatch=0 total_unreliable=0 total_evaluated=0 rundir="" n_written=0 aborted=0
4299
4546
  # Computed ONCE per batch, not per fixture row - see _eval_emit_result's comment.
4300
- local batch_cliv="" batch_flowv="" batch_grsha=""
4547
+ local batch_cliv="" batch_flowv="" batch_grsha="" prompt_sha_acc="" prompt_sha_enforced=0
4301
4548
  if [ "$replay_mode" -eq 1 ]; then
4302
4549
  # Replay wrap: skip probe SKIP-exit and _eval_cli_version only. Still compute
4303
4550
  # batch_grsha (staleness hard-fail) and _flow_version. Nonce from fixture, never _eval_nonce.
@@ -4320,12 +4567,17 @@ function cmd_eval {
4320
4567
  echo " recorded=$rec_sha current=$batch_grsha"
4321
4568
  return 1
4322
4569
  fi
4570
+ if grep -q '^prompt_sha\.' "$EVAL_REPLAY_DIR/meta" 2>/dev/null; then
4571
+ prompt_sha_enforced=1
4572
+ fi
4323
4573
  echo "replay: keyless artifact batch (nonce from fixture, zero live calls)"
4324
4574
  else
4325
- local probe; probe="$(_eval_probe)"
4575
+ _eval_isolated_guard || return 1
4576
+ local probe; probe="$(_eval_isolated_run _eval_probe)"
4326
4577
  case "$probe" in
4327
4578
  absent)
4328
- echo "SKIP: 'claude' CLI not found on PATH - eval needs it to run the LLM judge. Zero calls made."
4579
+ _eval_skip_unmeasured "eval"
4580
+ echo "semantic layer unmeasured on this host"
4329
4581
  return 0 ;;
4330
4582
  fail)
4331
4583
  echo "SKIP: 'claude' CLI is present but the sentinel probe did not come back clean (one minimal"
@@ -4425,13 +4677,36 @@ function cmd_eval {
4425
4677
  continue
4426
4678
  fi
4427
4679
 
4680
+ local psha
4681
+ psha="$(_eval_prompt_sha "$promptfile")"
4682
+ if [ "$replay_mode" -eq 1 ] && [ "$prompt_sha_enforced" -eq 1 ]; then
4683
+ local rec_psha
4684
+ rec_psha="$(_eval_replay_meta_get "prompt_sha.$fid")" || rec_psha=""
4685
+ if [ -z "$rec_psha" ] || [ "$rec_psha" != "$psha" ]; then
4686
+ echo "eval --replay: fixtures stale — prompt assembly changed; re-record live per ADR re-baseline rule"
4687
+ echo " recorded=$rec_psha current=$psha fixture=$fid"
4688
+ rm -rf "$rundir" 2>/dev/null
4689
+ trap - INT TERM
4690
+ return 1
4691
+ fi
4692
+ fi
4693
+ if [ "$record_mode" -eq 1 ]; then
4694
+ prompt_sha_acc="${prompt_sha_acc}prompt_sha.${fid}=${psha}"$'\n'
4695
+ fi
4696
+
4428
4697
  total_evaluated=$((total_evaluated + 1))
4429
- local flag_count=0 pass_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false
4698
+ local flag_count=0 pass_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false vote_timed_out=false
4430
4699
  # Filename-safe fixture id for raw-capture paths - manifest fid is only tr -d '\r'-cleaned
4431
4700
  # (the v1 trust-boundary is READ-side only, but we are now WRITING keyed by it), so a
4432
4701
  # hand-edited or FLOW_EVAL_MANIFEST-overridden manifest cannot traverse out of eval-raw/.
4433
4702
  local fid_safe; fid_safe="$(printf '%s' "$fid" | tr -c 'A-Za-z0-9' '-' | sed -E 's/-+/-/g; s/^-//; s/-$//')"
4434
4703
  [ -z "$fid_safe" ] && fid_safe="_"
4704
+ if [ "$replay_mode" -eq 0 ]; then
4705
+ local rawdir_prompt="$LOG_DIR/eval-raw/$nonce"
4706
+ if mkdir -p "$rawdir_prompt" 2>/dev/null; then
4707
+ cp "$promptfile" "$rawdir_prompt/${fid_safe}.prompt" 2>/dev/null || true
4708
+ fi
4709
+ fi
4435
4710
  while [ "$i" -le "$n" ]; do
4436
4711
  local raw rc v raw1="" rc1=0 err1="" err2=""
4437
4712
  # Attempt 1 - stderr captured at the OUTER redirection so it never enters the sh -c
@@ -4442,11 +4717,12 @@ function cmd_eval {
4442
4717
  raw="$(_eval_replay_vote "$fid" "$i")" || { rm -rf "$rundir" 2>/dev/null; trap - INT TERM; return 1; }
4443
4718
  rc=0
4444
4719
  else
4445
- raw="$(_eval_engine_run "$promptfile" "$timeout" 2>"$err1")"; rc=$?
4720
+ raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>"$err1")"; rc=$?
4446
4721
  fi
4447
4722
  raw1="$raw"; rc1="$rc"
4448
4723
  if [ "$rc" -eq 124 ]; then
4449
4724
  v=INVALID
4725
+ vote_timed_out=true
4450
4726
  else
4451
4727
  v="$(_eval_parse_verdict "$raw" "$nonce")"
4452
4728
  [ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"
@@ -4477,10 +4753,10 @@ function cmd_eval {
4477
4753
  raw="$(_eval_replay_vote "$fid" "$i")" || { rm -rf "$rundir" 2>/dev/null; trap - INT TERM; return 1; }
4478
4754
  rc=0
4479
4755
  else
4480
- raw="$(_eval_engine_run "$promptfile" "$timeout" 2>"$err2")"; rc=$?
4756
+ raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>"$err2")"; rc=$?
4481
4757
  fi
4482
4758
  retries_sum=$((retries_sum + 1))
4483
- if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_parse_verdict "$raw" "$nonce")"; fi
4759
+ if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_parse_verdict "$raw" "$nonce")"; fi
4484
4760
  [ "$model" = "unknown" ] && [ "$rc" -ne 124 ] && model="$(_eval_parse_model "$raw")"
4485
4761
  local rl2; rl2="$(_eval_parse_rate_limited "$raw")"
4486
4762
  [ "$rl2" = "true" ] && vote_rate_limited=true
@@ -4555,7 +4831,7 @@ function cmd_eval {
4555
4831
  fi
4556
4832
 
4557
4833
  if [ "$replay_mode" -eq 0 ]; then
4558
- _eval_emit_result "$nonce" "$fid" "$fstage" "$fexpected" "$verdict" "$match" "$flag_count" "$pass_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_grsha" "$retries_sum" "$vote_rate_limited"
4834
+ _eval_emit_result "$nonce" "$fid" "$fstage" "$fexpected" "$verdict" "$match" "$flag_count" "$pass_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_grsha" "$retries_sum" "$vote_rate_limited" "$psha" "$vote_timed_out"
4559
4835
  fi
4560
4836
  n_written=$((n_written + 1))
4561
4837
 
@@ -4587,7 +4863,7 @@ function cmd_eval {
4587
4863
  _eval_emit_batch_marker "$nonce" done "$n_written"
4588
4864
  fi
4589
4865
  if [ "$record_mode" -eq 1 ] && [ "$aborted" -eq 0 ]; then
4590
- _eval_replay_write_meta "$nonce" "$batch_grsha" "${model:-unknown}" "$n"
4866
+ _eval_replay_write_meta "$nonce" "$batch_grsha" "${model:-unknown}" "$n" "$prompt_sha_acc"
4591
4867
  fi
4592
4868
 
4593
4869
  echo