@manhquy/flow-skill 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +5 -5
- package/README_VN.md +5 -5
- package/package.json +1 -1
- package/skills/flow/SKILL.md +108 -219
- package/skills/flow/_templates/01-research.md +3 -2
- package/skills/flow/_templates/02-scope.md +2 -0
- package/skills/flow/_templates/03-prd.md +2 -1
- package/skills/flow/_templates/05-contract.md +1 -0
- package/skills/flow/_templates/card.md +5 -1
- package/skills/flow/law/CLAUDE.md +6 -5
- package/skills/flow/law/CODING.md +37 -0
- package/skills/flow/references/agent-detection.md +12 -9
- package/skills/flow/references/agent-stage-mapping.md +47 -12
- package/skills/flow/references/auto-run.md +38 -7
- package/skills/flow/references/clarify.md +7 -7
- package/skills/flow/references/command-dispatch.md +51 -51
- package/skills/flow/references/concierge.md +2 -2
- package/skills/flow/references/debt-and-halts.md +20 -5
- package/skills/flow/references/gate-00.md +10 -0
- package/skills/flow/references/gate-01.md +18 -0
- package/skills/flow/references/gate-02.md +16 -0
- package/skills/flow/references/gate-03.md +10 -0
- package/skills/flow/references/gate-04.md +12 -0
- package/skills/flow/references/gate-05.md +33 -0
- package/skills/flow/references/gate-assess.md +13 -0
- package/skills/flow/references/gate-card.md +25 -0
- package/skills/flow/references/gate-consistency.md +24 -0
- package/skills/flow/references/gate-constitution.md +19 -0
- package/skills/flow/references/gate-debt.md +5 -0
- package/skills/flow/references/gate-eval.md +54 -3
- package/skills/flow/references/gate-examples.md +78 -0
- package/skills/flow/references/gate-rules.md +40 -241
- package/skills/flow/references/gate-shared.md +37 -0
- package/skills/flow/references/ground-truth-gates.md +1 -1
- package/skills/flow/references/host-agnostic-parallel.md +54 -0
- package/skills/flow/references/mode-work.md +13 -10
- package/skills/flow/runner/flow.sh +334 -58
- package/skills-manifest.json +17 -2
|
@@ -171,6 +171,7 @@ current_stage_idx() { # highest CONTIGUOUS-from-00 stage index, or -1 if 00 miss
|
|
|
171
171
|
# Prints violations; returns 0 = clean, 1 = violations found.
|
|
172
172
|
scan_gate() {
|
|
173
173
|
local file="$1" found=0 unchecked fills
|
|
174
|
+
_MIDTIER_FAILS=0
|
|
174
175
|
if [ ! -f "$file" ]; then
|
|
175
176
|
echo " missing file: $file"
|
|
176
177
|
return 1
|
|
@@ -187,9 +188,132 @@ scan_gate() {
|
|
|
187
188
|
printf '%s\n' "$fills" | sed 's/^/ L/'
|
|
188
189
|
found=1
|
|
189
190
|
fi
|
|
191
|
+
case "$(basename "$file")" in
|
|
192
|
+
01-research.md|02-scope.md|03-prd.md|05-contract.md)
|
|
193
|
+
if ! _scan_midtier "$file"; then
|
|
194
|
+
found=1
|
|
195
|
+
fi
|
|
196
|
+
;;
|
|
197
|
+
esac
|
|
190
198
|
return $found
|
|
191
199
|
}
|
|
192
200
|
|
|
201
|
+
# Materialize a ## heading body into stdout (one awk; caller assigns to a var).
|
|
202
|
+
# Copy of cmd_clarify's heading walk: start at heading, stop at the next ##.
|
|
203
|
+
_heading_slice() { # $1=file $2=heading prefix after "## "
|
|
204
|
+
local file="$1" heading="$2"
|
|
205
|
+
awk -v h="$heading" '
|
|
206
|
+
{
|
|
207
|
+
line = $0
|
|
208
|
+
sub(/\r$/, "", line)
|
|
209
|
+
}
|
|
210
|
+
line ~ ("^##[[:space:]]+" h) { f=1; next }
|
|
211
|
+
/^##[[:space:]]/ { f=0 }
|
|
212
|
+
f { print line }
|
|
213
|
+
' "$file"
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
_midtier_hit() {
|
|
217
|
+
echo " [x] $1"
|
|
218
|
+
_MIDTIER_FAILS=$((_MIDTIER_FAILS + 1))
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
_midtier_entry_n() { # stdin = ## What exists already -> numbered or bullet entries
|
|
222
|
+
awk '
|
|
223
|
+
/^[[:space:]]*[0-9]+\./ { c++; next }
|
|
224
|
+
/^[[:space:]]*- / { c++ }
|
|
225
|
+
END { print c + 0 }
|
|
226
|
+
'
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
_midtier_l_above_a() { # stdin = ## Features in v1 body; exit 1 if a joined bullet is L + B/C
|
|
230
|
+
awk '
|
|
231
|
+
function flush() {
|
|
232
|
+
if (buf == "") return
|
|
233
|
+
low = tolower(buf)
|
|
234
|
+
hasL = (low ~ /impact[[:space:]]*:?[[:space:]]*l([^a-z]|$)/ || low ~ /(^|[^a-z])l-impact/)
|
|
235
|
+
hasBC = (low ~ /grade[[:space:]]*:?[[:space:]]*[bc]([^a-z]|$)/)
|
|
236
|
+
if (hasL && hasBC) bad++
|
|
237
|
+
buf = ""
|
|
238
|
+
}
|
|
239
|
+
/^[[:space:]]*- / { flush(); buf = $0; next }
|
|
240
|
+
/^[[:space:]]*$/ { flush(); next }
|
|
241
|
+
buf != "" { buf = buf " " $0; next }
|
|
242
|
+
{ flush() }
|
|
243
|
+
END { flush(); exit (bad > 0) ? 1 : 0 }
|
|
244
|
+
'
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
_midtier_adjectives() { # stdin = Features or NFR body; print first listed adjective
|
|
248
|
+
awk '
|
|
249
|
+
/^[[:space:]]*- \[[ xX]\]/ { next }
|
|
250
|
+
{
|
|
251
|
+
line = tolower($0)
|
|
252
|
+
n = split("fast secure intuitive robust prominent", w, " ")
|
|
253
|
+
for (i = 1; i <= n; i++) {
|
|
254
|
+
if (line ~ "(^|[^a-z])" w[i] "([^a-z]|$)") { print w[i]; exit }
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
'
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
_midtier_od_n() { # stdin = ## Open decisions body -> markdown bullet count (checked or not)
|
|
261
|
+
awk '/^[[:space:]]*- / { c++ } END { print c + 0 }'
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
_midtier_open_decisions() { # $1=file
|
|
265
|
+
local body n
|
|
266
|
+
body="$(_heading_slice "$1" "Open decisions")"
|
|
267
|
+
n="$(printf '%s\n' "$body" | _midtier_od_n)"
|
|
268
|
+
if [ "$n" -gt 5 ]; then
|
|
269
|
+
_midtier_hit "more than 5 ## Open decisions bullets ($n)"
|
|
270
|
+
fi
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
_scan_midtier() { # $1=file; prints [x] lines; return 1 if any hit
|
|
274
|
+
local file="$1" base body n pt adj
|
|
275
|
+
base="$(basename "$file")"
|
|
276
|
+
case "$base" in
|
|
277
|
+
01-research.md)
|
|
278
|
+
if [ -f "$PROJECT_TYPE_FILE" ]; then
|
|
279
|
+
pt="$(tr -d '\r' < "$PROJECT_TYPE_FILE" | awk 'NF{print; exit}')"
|
|
280
|
+
if [ "$pt" = "web" ]; then
|
|
281
|
+
body="$(_heading_slice "$file" "What exists already")"
|
|
282
|
+
n="$(printf '%s\n' "$body" | _midtier_entry_n)"
|
|
283
|
+
if [ "$n" -lt 3 ]; then
|
|
284
|
+
_midtier_hit "fewer than 3 entries under ## What exists already (typed web, got $n)"
|
|
285
|
+
fi
|
|
286
|
+
fi
|
|
287
|
+
fi
|
|
288
|
+
;;
|
|
289
|
+
02-scope.md)
|
|
290
|
+
body="$(_heading_slice "$file" "Features in v1")"
|
|
291
|
+
if ! printf '%s\n' "$body" | _midtier_l_above_a; then
|
|
292
|
+
_midtier_hit "L-impact feature above grade A in ## Features in v1"
|
|
293
|
+
fi
|
|
294
|
+
_midtier_open_decisions "$file"
|
|
295
|
+
;;
|
|
296
|
+
03-prd.md)
|
|
297
|
+
body="$(_heading_slice "$file" "Features")"
|
|
298
|
+
adj="$(printf '%s\n' "$body" | _midtier_adjectives)"
|
|
299
|
+
if [ -n "$adj" ]; then
|
|
300
|
+
_midtier_hit "unquantified adjective '$adj' in ## Features"
|
|
301
|
+
else
|
|
302
|
+
body="$(_heading_slice "$file" "Non-functional requirements")"
|
|
303
|
+
adj="$(printf '%s\n' "$body" | _midtier_adjectives)"
|
|
304
|
+
if [ -n "$adj" ]; then
|
|
305
|
+
_midtier_hit "unquantified adjective '$adj' in ## Non-functional requirements"
|
|
306
|
+
fi
|
|
307
|
+
fi
|
|
308
|
+
_midtier_open_decisions "$file"
|
|
309
|
+
;;
|
|
310
|
+
05-contract.md)
|
|
311
|
+
_midtier_open_decisions "$file"
|
|
312
|
+
;;
|
|
313
|
+
esac
|
|
314
|
+
[ "$_MIDTIER_FAILS" -eq 0 ]
|
|
315
|
+
}
|
|
316
|
+
|
|
193
317
|
_python() {
|
|
194
318
|
# prefer python3, and only accept an interpreter that is actually Python 3.x (the harness +
|
|
195
319
|
# repo_map.py are py3-only — a py2 `python` must NOT be selected or it fails silently).
|
|
@@ -437,6 +561,18 @@ planning_complete() { # 0 = yes: each stage is clean, OR debt-skipped (a skipped
|
|
|
437
561
|
return 0
|
|
438
562
|
}
|
|
439
563
|
|
|
564
|
+
# Files exist (or debt-skipped). Used by project-type lock so a midtier-dirty
|
|
565
|
+
# 01 cannot unlock a type flip after stages 00-05 are on disk.
|
|
566
|
+
planning_artifacts_present() {
|
|
567
|
+
local s
|
|
568
|
+
for s in $STAGES; do
|
|
569
|
+
if [ ! -f "$FLOW_DIR/$s.md" ]; then
|
|
570
|
+
stage_skipped "$s" || return 1
|
|
571
|
+
fi
|
|
572
|
+
done
|
|
573
|
+
return 0
|
|
574
|
+
}
|
|
575
|
+
|
|
440
576
|
# project type (web|cli|library|skill); absent => web. Adapts done-evidence + guidance.
|
|
441
577
|
get_project_type() {
|
|
442
578
|
local t; t="$(cat "$PROJECT_TYPE_FILE" 2>/dev/null | tr -d '\r' | awk 'NF{print; exit}')"
|
|
@@ -759,7 +895,7 @@ cmd_status() {
|
|
|
759
895
|
echo " auto: $_ast$([ "$_ast" = ACTIVE ] && echo " (receipts enforced)" || true)"
|
|
760
896
|
lock_warn
|
|
761
897
|
echo
|
|
762
|
-
|
|
898
|
+
_emit_next
|
|
763
899
|
echo
|
|
764
900
|
if [ -f "$FLOW_DIR/00-inspect.md" ]; then
|
|
765
901
|
if scan_gate "$FLOW_DIR/00-inspect.md" >/dev/null 2>&1; then
|
|
@@ -856,7 +992,9 @@ _resume_valid_lines() {
|
|
|
856
992
|
cmd_resume() {
|
|
857
993
|
local idx; idx="$(current_stage_idx)"
|
|
858
994
|
if [ "$idx" -lt 0 ]; then
|
|
859
|
-
|
|
995
|
+
_na="$(_next_action)"
|
|
996
|
+
echo "nothing to resume - $_na"
|
|
997
|
+
echo "NEXT_VERB=$(_next_verb_from "$_na")"
|
|
860
998
|
return 0
|
|
861
999
|
fi
|
|
862
1000
|
|
|
@@ -874,7 +1012,7 @@ cmd_resume() {
|
|
|
874
1012
|
echo
|
|
875
1013
|
_gate_state_brief "$idx"
|
|
876
1014
|
echo
|
|
877
|
-
|
|
1015
|
+
_emit_next
|
|
878
1016
|
return 0
|
|
879
1017
|
fi
|
|
880
1018
|
|
|
@@ -966,7 +1104,7 @@ cmd_resume() {
|
|
|
966
1104
|
echo
|
|
967
1105
|
|
|
968
1106
|
# ---- NEXT (shared with `status` via _next_action - the two verbs can never disagree) ----
|
|
969
|
-
|
|
1107
|
+
_emit_next
|
|
970
1108
|
}
|
|
971
1109
|
|
|
972
1110
|
cmd_next() {
|
|
@@ -1005,7 +1143,7 @@ cmd_next() {
|
|
|
1005
1143
|
# attribute the failing event to THIS stage (so usage top-fail-stage + propose can see it)
|
|
1006
1144
|
# and record WHICH checks failed, so a chronically-failing stage is diagnosable.
|
|
1007
1145
|
FLOW_LOG_STAGE_TO="$cur"
|
|
1008
|
-
FLOW_LAST_GATE_FAIL="fill:$(grep -c '\[FILL' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0),unchecked:$(grep -cE '^[[:space:]]*- \[ \]' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0)"
|
|
1146
|
+
FLOW_LAST_GATE_FAIL="fill:$(grep -c '\[FILL' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0),unchecked:$(grep -cE '^[[:space:]]*- \[ \]' "$FLOW_DIR/$cur.md" 2>/dev/null || echo 0),midtier:${_MIDTIER_FAILS:-0}"
|
|
1009
1147
|
echo
|
|
1010
1148
|
echo "Fix the above, then run '/flow next' again. (Kill at a gate is also valid.)"
|
|
1011
1149
|
return 1
|
|
@@ -1179,6 +1317,27 @@ _next_action() {
|
|
|
1179
1317
|
echo "run '/flow check' then ship per stage 09"
|
|
1180
1318
|
}
|
|
1181
1319
|
|
|
1320
|
+
# Closed advisory enum from _next_action prose only. Hosts must not auto-exec auto/skip.
|
|
1321
|
+
_next_verb_from() { # $1 = _next_action text
|
|
1322
|
+
local a="$1"
|
|
1323
|
+
case "$a" in
|
|
1324
|
+
*"fix gate:"*) echo fix-gate ;;
|
|
1325
|
+
*"/flow card start"*) echo card-start ;;
|
|
1326
|
+
*"/flow card"*) echo card ;;
|
|
1327
|
+
*"/flow check"*) echo check ;;
|
|
1328
|
+
*"/flow next"*) echo next ;;
|
|
1329
|
+
*"in flight"*) echo card-start ;;
|
|
1330
|
+
*) echo none ;;
|
|
1331
|
+
esac
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
_emit_next() {
|
|
1335
|
+
local a
|
|
1336
|
+
a="$(_next_action)"
|
|
1337
|
+
echo "NEXT -> $a"
|
|
1338
|
+
echo "NEXT_VERB=$(_next_verb_from "$a")"
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1182
1341
|
_set_card_status() { # $1=file $2=value -> rewrite the ^status: line (portable substitute; temp+mv)
|
|
1183
1342
|
local f="$1" v="$2"
|
|
1184
1343
|
sed "s/^status:.*/status: $v/" "$f" > "$f.tmp" 2>/dev/null || { rm -f "$f.tmp"; return 1; }
|
|
@@ -1590,12 +1749,13 @@ cmd_project_type() {
|
|
|
1590
1749
|
fi
|
|
1591
1750
|
case "$arg" in
|
|
1592
1751
|
web|cli|library|skill)
|
|
1593
|
-
# After planning
|
|
1752
|
+
# After planning artifacts exist AND an explicit type file exists, refuse silent
|
|
1594
1753
|
# type flips unless FLOW_FORCE=1 (D6). Default-web with no file is not "locked".
|
|
1595
|
-
|
|
1754
|
+
# Gate cleanliness is NOT required: a midtier-dirty 01 must not unlock a flip.
|
|
1755
|
+
if planning_artifacts_present 2>/dev/null && [ -f "$PROJECT_TYPE_FILE" ]; then
|
|
1596
1756
|
local cur; cur="$(get_project_type)"
|
|
1597
1757
|
if [ -n "$cur" ] && [ "$cur" != "$arg" ] && [ "${FLOW_FORCE:-0}" != "1" ]; then
|
|
1598
|
-
echo "FAIL: project type is locked to '$cur'
|
|
1758
|
+
echo "FAIL: project type is locked to '$cur' once planning artifacts exist (would change done-evidence lens)."
|
|
1599
1759
|
echo " re-set with FLOW_FORCE=1 if intentional, and record a DEBT.md line for the change."
|
|
1600
1760
|
return 1
|
|
1601
1761
|
fi
|
|
@@ -2302,6 +2462,14 @@ cmd_doctor() {
|
|
|
2302
2462
|
command -v cargo >/dev/null 2>&1 && echo " cargo: ok (optional Rust harness power-path)" || true
|
|
2303
2463
|
[ -d "$TEMPLATE_DIR" ] && echo " templates: ok" || { echo " templates: MISSING ($TEMPLATE_DIR)"; ok=0; }
|
|
2304
2464
|
[ -f "$HARNESS_PY" ] && echo " harness: present ($HARNESS_PY)" || echo " harness: absent (durable layer off)"
|
|
2465
|
+
if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then
|
|
2466
|
+
echo " timeout: full (timeout|gtimeout present)"
|
|
2467
|
+
else
|
|
2468
|
+
case "$(uname -s 2>/dev/null)" in
|
|
2469
|
+
Darwin) echo " timeout: absent (no timeout/gtimeout; live eval REFUSED)" ;;
|
|
2470
|
+
*) echo " timeout: partial (watchdog fallback; live eval not GNU-bounded)" ;;
|
|
2471
|
+
esac
|
|
2472
|
+
fi
|
|
2305
2473
|
echo
|
|
2306
2474
|
if [ "$ok" -eq 1 ]; then
|
|
2307
2475
|
echo "READY. Gate engine works$([ -n "$py" ] && echo '; durable layer on.' || echo '; durable layer off (no python).')"
|
|
@@ -3154,17 +3322,26 @@ _eval_heading_pattern() {
|
|
|
3154
3322
|
esac
|
|
3155
3323
|
}
|
|
3156
3324
|
|
|
3157
|
-
# Extract one stage's ritual text
|
|
3158
|
-
#
|
|
3159
|
-
#
|
|
3325
|
+
# Extract one stage's ritual text from the mapped file (01→gate-01.md, 02→gate-02.md,
|
|
3326
|
+
# card→gate-card.md): from its heading line up to (not including) the next '## '
|
|
3327
|
+
# heading, or EOF. Prints nothing on a bad stage/missing file - caller MUST assert
|
|
3328
|
+
# non-empty before any billable call (a silent empty extraction would judge against
|
|
3329
|
+
# no rule at all).
|
|
3160
3330
|
_eval_extract_section() { # $1 = stage (01|02|card)
|
|
3161
|
-
local pat
|
|
3162
|
-
|
|
3331
|
+
local pat src
|
|
3332
|
+
pat="$(_eval_heading_pattern "$1")" || return 1
|
|
3333
|
+
case "$1" in
|
|
3334
|
+
01) src="$SCRIPT_DIR/../references/gate-01.md" ;;
|
|
3335
|
+
02) src="$SCRIPT_DIR/../references/gate-02.md" ;;
|
|
3336
|
+
card) src="$SCRIPT_DIR/../references/gate-card.md" ;;
|
|
3337
|
+
*) return 1 ;;
|
|
3338
|
+
esac
|
|
3339
|
+
[ -f "$src" ] || return 1
|
|
3163
3340
|
awk -v pat="$pat" '
|
|
3164
3341
|
$0 ~ pat { f=1; print; next }
|
|
3165
3342
|
f && /^## / { f=0 }
|
|
3166
3343
|
f { print }
|
|
3167
|
-
' "$
|
|
3344
|
+
' "$src"
|
|
3168
3345
|
}
|
|
3169
3346
|
|
|
3170
3347
|
# Usability probe: zero cost if `claude` is absent; exactly one minimal billable call if present.
|
|
@@ -3199,13 +3376,39 @@ _eval_engine_run() { # $1=promptfile $2=timeout-seconds -> stdout=raw json; retu
|
|
|
3199
3376
|
# command string above. Empirical: adding a `2> '$errfile'` inside the timed cmd breaks the
|
|
3200
3377
|
# watchdog-fallback path in `_run_with_timeout` on the timeout-less-PATH lane (macOS DEBT lane).
|
|
3201
3378
|
# Outer-scope redirection is exactly the same capture at zero cost to the timeout contract.
|
|
3379
|
+
# Opt-in isolated cwd for live eval (Approach D). Unset = today's cwd (byte-identical
|
|
3380
|
+
# call path). Set but not a directory = fail loud. Isolation is cd+env around existing
|
|
3381
|
+
# probe/engine calls — never argv, never a second engine. CLAUDE_CODE_SAFE_MODE=1 is
|
|
3382
|
+
# exported ONLY in the isolated subshell (official Claude env; keeps OAuth; blast radius
|
|
3383
|
+
# is the judge child). Do not invent a flow alias; do not set HOME or CLAUDE_CONFIG_DIR.
|
|
3384
|
+
_eval_isolated_guard() {
|
|
3385
|
+
[ -z "${FLOW_EVAL_ISOLATED_CWD:-}" ] && return 0
|
|
3386
|
+
[ -d "$FLOW_EVAL_ISOLATED_CWD" ] && return 0
|
|
3387
|
+
echo "FAIL: FLOW_EVAL_ISOLATED_CWD is set but is not a directory: $FLOW_EVAL_ISOLATED_CWD"
|
|
3388
|
+
return 1
|
|
3389
|
+
}
|
|
3390
|
+
_eval_isolated_run() {
|
|
3391
|
+
if [ -n "${FLOW_EVAL_ISOLATED_CWD:-}" ]; then
|
|
3392
|
+
( cd "$FLOW_EVAL_ISOLATED_CWD" && export CLAUDE_CODE_SAFE_MODE=1 && "$@" )
|
|
3393
|
+
else
|
|
3394
|
+
"$@"
|
|
3395
|
+
fi
|
|
3396
|
+
}
|
|
3397
|
+
|
|
3202
3398
|
|
|
3203
|
-
# Build the judge prompt for one fixture.
|
|
3204
|
-
#
|
|
3399
|
+
# Build the judge prompt for one fixture. The challenge block is the extract output
|
|
3400
|
+
# (already the judge bytes; `_eval_gate_rules_sha` is not stdin). Returns 1 if empty.
|
|
3205
3401
|
_eval_build_prompt() { # $1=outfile $2=stage $3=artifact-file $4=nonce
|
|
3206
|
-
local outfile="$1" stage="$2" artifact="$3" nonce="$4" section
|
|
3402
|
+
local outfile="$1" stage="$2" artifact="$3" nonce="$4" section shared
|
|
3207
3403
|
section="$(_eval_extract_section "$stage")" || return 1
|
|
3208
3404
|
[ -n "$section" ] || return 1
|
|
3405
|
+
# Shared OD/authority lives in 02/03/05 challenges. Extract maps 01/02/card only;
|
|
3406
|
+
# appending shared onto 01/card lets OD rules false-FLAG research/evidence.
|
|
3407
|
+
if [ "$stage" = "02" ]; then
|
|
3408
|
+
shared="$SCRIPT_DIR/../references/gate-shared.md"
|
|
3409
|
+
[ -f "$shared" ] && section="$section
|
|
3410
|
+
$(cat "$shared")"
|
|
3411
|
+
fi
|
|
3209
3412
|
{
|
|
3210
3413
|
printf 'You are reviewing a build-process artifact against the quality-gate challenge below.\n'
|
|
3211
3414
|
printf 'Read the challenge, then the artifact, then decide honestly: does the artifact\n'
|
|
@@ -3284,11 +3487,34 @@ _eval_nonce_epoch() { # $1=run_id -> epoch seconds (or empty)
|
|
|
3284
3487
|
printf '%s' "$1" | awk -F- 'NF>=2 { print $(NF-1) }'
|
|
3285
3488
|
}
|
|
3286
3489
|
|
|
3287
|
-
# CRLF-normalized
|
|
3288
|
-
#
|
|
3490
|
+
# CRLF-normalized hash of the explicit gate-rules corpus (index + shared + 11
|
|
3491
|
+
# section files). Never glob `gate-*.md` (would pull gate-eval.md + gate-examples.md).
|
|
3492
|
+
# A raw-byte hash would differ per checkout line-endings (git autocrlf).
|
|
3289
3493
|
_eval_gate_rules_sha() {
|
|
3290
|
-
|
|
3291
|
-
|
|
3494
|
+
local ref="$SCRIPT_DIR/../references"
|
|
3495
|
+
[ -f "$ref/gate-rules.md" ] || { printf 'unknown'; return; }
|
|
3496
|
+
{
|
|
3497
|
+
cat "$ref/gate-rules.md"
|
|
3498
|
+
[ -f "$ref/gate-shared.md" ] && cat "$ref/gate-shared.md"
|
|
3499
|
+
[ -f "$ref/gate-assess.md" ] && cat "$ref/gate-assess.md"
|
|
3500
|
+
[ -f "$ref/gate-00.md" ] && cat "$ref/gate-00.md"
|
|
3501
|
+
[ -f "$ref/gate-01.md" ] && cat "$ref/gate-01.md"
|
|
3502
|
+
[ -f "$ref/gate-02.md" ] && cat "$ref/gate-02.md"
|
|
3503
|
+
[ -f "$ref/gate-03.md" ] && cat "$ref/gate-03.md"
|
|
3504
|
+
[ -f "$ref/gate-04.md" ] && cat "$ref/gate-04.md"
|
|
3505
|
+
[ -f "$ref/gate-05.md" ] && cat "$ref/gate-05.md"
|
|
3506
|
+
[ -f "$ref/gate-card.md" ] && cat "$ref/gate-card.md"
|
|
3507
|
+
[ -f "$ref/gate-consistency.md" ] && cat "$ref/gate-consistency.md"
|
|
3508
|
+
[ -f "$ref/gate-constitution.md" ] && cat "$ref/gate-constitution.md"
|
|
3509
|
+
[ -f "$ref/gate-debt.md" ] && cat "$ref/gate-debt.md"
|
|
3510
|
+
} | tr -d '\r' | cksum | awk '{print $1}'
|
|
3511
|
+
}
|
|
3512
|
+
|
|
3513
|
+
# CRLF-normalized assembler-output hash (same recipe as _eval_gate_rules_sha). Covers
|
|
3514
|
+
# prompt-file bytes only — not host CLAUDE.md. $1=prompt file.
|
|
3515
|
+
_eval_prompt_sha() {
|
|
3516
|
+
[ -f "$1" ] || { printf 'unknown'; return; }
|
|
3517
|
+
tr -d '\r' < "$1" | cksum | awk '{print $1}'
|
|
3292
3518
|
}
|
|
3293
3519
|
|
|
3294
3520
|
# `claude --version` is the CLI version (e.g. "2.1.201 (Claude Code)"), NOT a model id - kept
|
|
@@ -3317,9 +3543,10 @@ _eval_replay_meta_get() { # $1=key -> value (empty + return 1 if missing)
|
|
|
3317
3543
|
return 1
|
|
3318
3544
|
}
|
|
3319
3545
|
|
|
3320
|
-
_eval_replay_write_meta() { # $1=nonce $2=gate_rules_sha $3=model $4=n
|
|
3546
|
+
_eval_replay_write_meta() { # $1=nonce $2=gate_rules_sha $3=model $4=n [$5=prompt_sha.<fid>= lines]
|
|
3321
3547
|
mkdir -p "$EVAL_REPLAY_DIR" || return 1
|
|
3322
|
-
printf 'nonce=%s\ngate_rules_sha=%s\nmodel=%s\nn=%s\n' "$1" "$2" "$3" "$4" > "$EVAL_REPLAY_DIR/meta"
|
|
3548
|
+
printf 'nonce=%s\ngate_rules_sha=%s\nmodel=%s\nn=%s\n' "$1" "$2" "$3" "$4" > "$EVAL_REPLAY_DIR/meta" || return 1
|
|
3549
|
+
[ -n "${5:-}" ] && printf '%s' "$5" >> "$EVAL_REPLAY_DIR/meta"
|
|
3323
3550
|
}
|
|
3324
3551
|
|
|
3325
3552
|
# Persist one stripped verdict line only — never a JSON envelope (session_id/cwd stay out).
|
|
@@ -3366,17 +3593,18 @@ _eval_emit_batch_marker() { # $1=run_id $2=start|done $3=n_expected-or-n_written
|
|
|
3366
3593
|
# cli_version/flow_version/gate_rules_sha are passed in (computed ONCE per batch by the caller,
|
|
3367
3594
|
# not per fixture row) - they never change mid-batch, and recomputing per-row was both wasteful
|
|
3368
3595
|
# (an extra subprocess per fixture) and the original vector for the stdin-consumption bug above.
|
|
3369
|
-
_eval_emit_result() { # $1..$14 as before + $15=retries $16=rate_limited
|
|
3596
|
+
_eval_emit_result() { # $1..$14 as before + $15=retries $16=rate_limited $17=prompt_sha $18=timed_out
|
|
3370
3597
|
local run_id="$1" fid="$2" stage="$3" expected="$4" verdict="$5" match="$6" flag="$7" pass="$8" invalid="$9" n="${10}" model="${11}" cliv="${12}" flowv="${13}" grsha="${14}"
|
|
3371
|
-
local retries="${15:-0}" ratel="${16:-false}"
|
|
3598
|
+
local retries="${15:-0}" ratel="${16:-false}" psha="${17:-}" timedout="${18:-false}"
|
|
3372
3599
|
case "$retries" in ''|*[!0-9]*) retries=0 ;; esac
|
|
3373
3600
|
case "$ratel" in true|false) : ;; *) ratel=false ;; esac
|
|
3601
|
+
case "$timedout" in true|false) : ;; *) timedout=false ;; esac
|
|
3374
3602
|
mkdir -p "$LOG_DIR" 2>/dev/null || true
|
|
3375
|
-
printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","stage":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"flag":%s,"pass":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","gate_rules_sha":"%s","retries":%s,"rate_limited":%s}\n' \
|
|
3603
|
+
printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","stage":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"flag":%s,"pass":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","gate_rules_sha":"%s","retries":%s,"rate_limited":%s,"prompt_sha":"%s","timed_out":%s}\n' \
|
|
3376
3604
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" "$(_now)" "$(_json_str "$run_id")" "$(_json_str "$fid")" \
|
|
3377
3605
|
"$(_json_str "$stage")" "$(_json_str "$expected")" "$(_json_str "$verdict")" "$(_json_str "$match")" \
|
|
3378
3606
|
"$flag" "$pass" "$invalid" "$n" "$(_json_str "$cliv")" "$(_json_str "$model")" \
|
|
3379
|
-
"$(_json_str "$flowv")" "$(_json_str "$grsha")" "$retries" "$ratel" >> "$EVAL_RESULTS_FILE" 2>/dev/null || true
|
|
3607
|
+
"$(_json_str "$flowv")" "$(_json_str "$grsha")" "$retries" "$ratel" "$(_json_str "$psha")" "$timedout" >> "$EVAL_RESULTS_FILE" 2>/dev/null || true
|
|
3380
3608
|
}
|
|
3381
3609
|
|
|
3382
3610
|
# Strip the noisy `system`/`init` envelope from a raw --output-format json line before persisting
|
|
@@ -3660,17 +3888,18 @@ _eval_routing_parse_action() {
|
|
|
3660
3888
|
[ -n "$action" ] && printf '%s' "$action" || echo INVALID
|
|
3661
3889
|
}
|
|
3662
3890
|
|
|
3663
|
-
_eval_routing_emit_result() { # $1=run_id $2=fid $3=expected $4=verdict $5=match $6..$9=votes
|
|
3891
|
+
_eval_routing_emit_result() { # $1=run_id $2=fid $3=expected $4=verdict $5=match $6..$9=votes $10=model $11=cliv $12=flowv $13=rulessha $14=retries $15=rate_limited $16=prompt_sha $17=timed_out
|
|
3664
3892
|
local run_id="$1" fid="$2" expected="$3" verdict="$4" match="$5" hit="$6" miss="$7" invalid="$8" n="$9"
|
|
3665
|
-
local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}"
|
|
3893
|
+
local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}" psha="${16:-}" timedout="${17:-false}"
|
|
3666
3894
|
case "$retries" in ''|*[!0-9]*) retries=0 ;; esac
|
|
3667
3895
|
case "$ratel" in true|false) : ;; *) ratel=false ;; esac
|
|
3896
|
+
case "$timedout" in true|false) : ;; *) timedout=false ;; esac
|
|
3668
3897
|
mkdir -p "$LOG_DIR" 2>/dev/null || true
|
|
3669
|
-
printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s}\n' \
|
|
3898
|
+
printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s,"prompt_sha":"%s","timed_out":%s}\n' \
|
|
3670
3899
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" "$(_now)" "$(_json_str "$run_id")" "$(_json_str "$fid")" \
|
|
3671
3900
|
"$(_json_str "$expected")" "$(_json_str "$verdict")" "$(_json_str "$match")" \
|
|
3672
3901
|
"$hit" "$miss" "$invalid" "$n" "$(_json_str "$cliv")" "$(_json_str "$model")" \
|
|
3673
|
-
"$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" >> "$EVAL_ROUTING_RESULTS_FILE" 2>/dev/null || true
|
|
3902
|
+
"$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" "$(_json_str "$psha")" "$timedout" >> "$EVAL_ROUTING_RESULTS_FILE" 2>/dev/null || true
|
|
3674
3903
|
}
|
|
3675
3904
|
|
|
3676
3905
|
_eval_routing_complete_run_ids() {
|
|
@@ -3744,6 +3973,9 @@ _eval_routing_print_scorecard() { # $1=run_id
|
|
|
3744
3973
|
'
|
|
3745
3974
|
}
|
|
3746
3975
|
|
|
3976
|
+
_eval_skip_unmeasured() { # $1 = kind (eval / routing eval / converge eval)
|
|
3977
|
+
echo "SKIP: 'claude' CLI not found on PATH - $1 needs it to run the LLM judge. Zero calls made."
|
|
3978
|
+
}
|
|
3747
3979
|
# NOTE: defined with the `function name { }` form (no parens) rather than this file's usual
|
|
3748
3980
|
# `name() { }` style, solely so the literal 4-byte substring the shipped verb name is built
|
|
3749
3981
|
# from never appears immediately followed by '(' in the source - a blind text-pattern security
|
|
@@ -3770,10 +4002,13 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
|
|
|
3770
4002
|
return 1
|
|
3771
4003
|
fi
|
|
3772
4004
|
|
|
3773
|
-
|
|
4005
|
+
_eval_isolated_guard || return 1
|
|
4006
|
+
|
|
4007
|
+
local probe; probe="$(_eval_isolated_run _eval_probe)"
|
|
3774
4008
|
case "$probe" in
|
|
3775
4009
|
absent)
|
|
3776
|
-
|
|
4010
|
+
_eval_skip_unmeasured "routing eval"
|
|
4011
|
+
echo "semantic layer unmeasured on this host"
|
|
3777
4012
|
return 0 ;;
|
|
3778
4013
|
fail)
|
|
3779
4014
|
echo "SKIP: 'claude' CLI is present but the sentinel probe did not come back clean (one minimal"
|
|
@@ -3838,14 +4073,18 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
|
|
|
3838
4073
|
continue
|
|
3839
4074
|
fi
|
|
3840
4075
|
|
|
4076
|
+
local psha
|
|
4077
|
+
psha="$(_eval_prompt_sha "$promptfile")"
|
|
4078
|
+
|
|
3841
4079
|
total_evaluated=$((total_evaluated + 1))
|
|
3842
|
-
local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false
|
|
4080
|
+
local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false vote_timed_out=false
|
|
3843
4081
|
while [ "$i" -le "$n" ]; do
|
|
3844
4082
|
local raw rc v raw1="" rc1=0
|
|
3845
|
-
raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
|
|
4083
|
+
raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
|
|
3846
4084
|
raw1="$raw"; rc1="$rc"
|
|
3847
4085
|
if [ "$rc" -eq 124 ]; then
|
|
3848
4086
|
v=INVALID
|
|
4087
|
+
vote_timed_out=true
|
|
3849
4088
|
else
|
|
3850
4089
|
v="$(_eval_routing_parse_action "$raw" "$nonce")"
|
|
3851
4090
|
[ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"
|
|
@@ -3857,9 +4096,9 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
|
|
|
3857
4096
|
echo " $fid: retrying vote $i after ${backoff}s (parse-INVALID on attempt 1)"
|
|
3858
4097
|
sleep "$backoff" 2>/dev/null || true
|
|
3859
4098
|
fi
|
|
3860
|
-
raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
|
|
4099
|
+
raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?
|
|
3861
4100
|
retries_sum=$((retries_sum + 1))
|
|
3862
|
-
if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_routing_parse_action "$raw" "$nonce")"; fi
|
|
4101
|
+
if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_routing_parse_action "$raw" "$nonce")"; fi
|
|
3863
4102
|
[ "$model" = "unknown" ] && [ "$rc" -ne 124 ] && model="$(_eval_parse_model "$raw")"
|
|
3864
4103
|
local rl2; rl2="$(_eval_parse_rate_limited "$raw")"
|
|
3865
4104
|
[ "$rl2" = "true" ] && vote_rate_limited=true
|
|
@@ -3897,7 +4136,7 @@ function cmd_eval_routing { # same arg shape as cmd_eval, called only when --sta
|
|
|
3897
4136
|
echo " $fid: MISS - expected '$fexpected' (hit=$hit_count miss=$miss_count invalid=$invalid_count)"
|
|
3898
4137
|
fi
|
|
3899
4138
|
|
|
3900
|
-
_eval_routing_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited"
|
|
4139
|
+
_eval_routing_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited" "$psha" "$vote_timed_out"
|
|
3901
4140
|
n_written=$((n_written + 1))
|
|
3902
4141
|
|
|
3903
4142
|
if [ "$total_evaluated" -eq 1 ] && [ "$verdict" = "UNRELIABLE" ] && [ "$keep_going" -eq 0 ]; then
|
|
@@ -4005,15 +4244,16 @@ _eval_converge_parse_verdict() { # $1=raw $2=nonce -> GAP|CONVERGED|INVALID
|
|
|
4005
4244
|
|
|
4006
4245
|
_eval_converge_emit_result() { # same field shape as _eval_routing_emit_result
|
|
4007
4246
|
local run_id="$1" fid="$2" expected="$3" verdict="$4" match="$5" hit="$6" miss="$7" invalid="$8" n="$9"
|
|
4008
|
-
local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}"
|
|
4247
|
+
local model="${10}" cliv="${11}" flowv="${12}" rulessha="${13}" retries="${14:-0}" ratel="${15:-false}" psha="${16:-}" timedout="${17:-false}"
|
|
4009
4248
|
case "$retries" in ''|*[!0-9]*) retries=0 ;; esac
|
|
4010
4249
|
case "$ratel" in true|false) : ;; *) ratel=false ;; esac
|
|
4250
|
+
case "$timedout" in true|false) : ;; *) timedout=false ;; esac
|
|
4011
4251
|
mkdir -p "$LOG_DIR" 2>/dev/null || true
|
|
4012
|
-
printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s}\n' \
|
|
4252
|
+
printf '{"ts":"%s","epoch_s":%s,"run_id":"%s","fixture":"%s","expected":"%s","verdict":"%s","match":"%s","votes":{"hit":%s,"miss":%s,"invalid":%s},"n":%s,"cli_version":"%s","model":"%s","flow_version":"%s","rules_sha":"%s","retries":%s,"rate_limited":%s,"prompt_sha":"%s","timed_out":%s}\n' \
|
|
4013
4253
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" "$(_now)" "$(_json_str "$run_id")" "$(_json_str "$fid")" \
|
|
4014
4254
|
"$(_json_str "$expected")" "$(_json_str "$verdict")" "$(_json_str "$match")" \
|
|
4015
4255
|
"$hit" "$miss" "$invalid" "$n" "$(_json_str "$cliv")" "$(_json_str "$model")" \
|
|
4016
|
-
"$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" >> "$EVAL_CONVERGE_RESULTS_FILE" 2>/dev/null || true
|
|
4256
|
+
"$(_json_str "$flowv")" "$(_json_str "$rulessha")" "$retries" "$ratel" "$(_json_str "$psha")" "$timedout" >> "$EVAL_CONVERGE_RESULTS_FILE" 2>/dev/null || true
|
|
4017
4257
|
}
|
|
4018
4258
|
|
|
4019
4259
|
_eval_converge_emit_batch_marker() { # $1=run_id $2=start|done $3=n
|
|
@@ -4075,9 +4315,11 @@ function cmd_eval_converge { # same arg shape as cmd_eval, called only when --st
|
|
|
4075
4315
|
return 1
|
|
4076
4316
|
fi
|
|
4077
4317
|
|
|
4078
|
-
|
|
4318
|
+
_eval_isolated_guard || return 1
|
|
4319
|
+
|
|
4320
|
+
local probe; probe="$(_eval_isolated_run _eval_probe)"
|
|
4079
4321
|
case "$probe" in
|
|
4080
|
-
absent)
|
|
4322
|
+
absent) _eval_skip_unmeasured "converge eval"; echo "semantic layer unmeasured on this host"; return 0 ;;
|
|
4081
4323
|
fail) echo "SKIP: 'claude' CLI present but the sentinel probe did not come back clean (one minimal probe call was made). Retry once 'claude -p' runs headless."; return 0 ;;
|
|
4082
4324
|
esac
|
|
4083
4325
|
|
|
@@ -4123,17 +4365,20 @@ function cmd_eval_converge { # same arg shape as cmd_eval, called only when --st
|
|
|
4123
4365
|
echo " $fid: FAIL - could not build converge prompt (converge.md/repo-dir/challenge missing)"; total_miss=$((total_miss + 1)); rm -rf "$rundir" 2>/dev/null; continue
|
|
4124
4366
|
fi
|
|
4125
4367
|
|
|
4368
|
+
local psha
|
|
4369
|
+
psha="$(_eval_prompt_sha "$promptfile")"
|
|
4370
|
+
|
|
4126
4371
|
total_evaluated=$((total_evaluated + 1))
|
|
4127
|
-
local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false
|
|
4372
|
+
local hit_count=0 miss_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false vote_timed_out=false
|
|
4128
4373
|
while [ "$i" -le "$n" ]; do
|
|
4129
4374
|
local raw rc v raw1="" rc1=0
|
|
4130
|
-
raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; raw1="$raw"; rc1="$rc"
|
|
4131
|
-
if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; [ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"; fi
|
|
4375
|
+
raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; raw1="$raw"; rc1="$rc"
|
|
4376
|
+
if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; [ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"; fi
|
|
4132
4377
|
local rl1; rl1="$(_eval_parse_rate_limited "$raw1")"; [ "$rl1" = "true" ] && vote_rate_limited=true
|
|
4133
4378
|
if [ "$v" = "INVALID" ] && [ "$rl1" != "true" ] && [ "$rc1" -ne 124 ]; then
|
|
4134
4379
|
if [ "$backoff" -gt 0 ]; then echo " $fid: retrying vote $i after ${backoff}s (parse-INVALID on attempt 1)"; sleep "$backoff" 2>/dev/null || true; fi
|
|
4135
|
-
raw="$(_eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; retries_sum=$((retries_sum + 1))
|
|
4136
|
-
if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; fi
|
|
4380
|
+
raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>/dev/null)"; rc=$?; retries_sum=$((retries_sum + 1))
|
|
4381
|
+
if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_converge_parse_verdict "$raw" "$nonce")"; fi
|
|
4137
4382
|
[ "$model" = "unknown" ] && [ "$rc" -ne 124 ] && model="$(_eval_parse_model "$raw")"
|
|
4138
4383
|
local rl2; rl2="$(_eval_parse_rate_limited "$raw")"; [ "$rl2" = "true" ] && vote_rate_limited=true
|
|
4139
4384
|
fi
|
|
@@ -4158,7 +4403,7 @@ function cmd_eval_converge { # same arg shape as cmd_eval, called only when --st
|
|
|
4158
4403
|
total_miss=$((total_miss + 1)); echo " $fid: MISS - expected '$fexpected' (hit=$hit_count miss=$miss_count invalid=$invalid_count)"
|
|
4159
4404
|
fi
|
|
4160
4405
|
|
|
4161
|
-
_eval_converge_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited"
|
|
4406
|
+
_eval_converge_emit_result "$nonce" "$fid" "$fexpected" "$verdict" "$match" "$hit_count" "$miss_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_rulessha" "$retries_sum" "$vote_rate_limited" "$psha" "$vote_timed_out"
|
|
4162
4407
|
n_written=$((n_written + 1))
|
|
4163
4408
|
|
|
4164
4409
|
if [ "$total_evaluated" -eq 1 ] && [ "$verdict" = "UNRELIABLE" ] && [ "$keep_going" -eq 0 ]; then
|
|
@@ -4189,6 +4434,8 @@ _eval_guard_unbounded_darwin() { # $1=replay_mode (0/1); return 1 = refused
|
|
|
4189
4434
|
local replay_mode="${1:-0}"
|
|
4190
4435
|
[ "$replay_mode" -eq 0 ] || return 0
|
|
4191
4436
|
[ "${FLOW_EVAL_UNBOUNDED:-}" = "1" ] && return 0
|
|
4437
|
+
# No judge binary => no stuck call to bound. Probe SKIP (unmeasured) owns this case.
|
|
4438
|
+
command -v claude >/dev/null 2>&1 || return 0
|
|
4192
4439
|
local darwin=0
|
|
4193
4440
|
if [ "${FLOW_EVAL_FORCE_DARWIN:-}" = "1" ]; then
|
|
4194
4441
|
darwin=1
|
|
@@ -4297,7 +4544,7 @@ function cmd_eval {
|
|
|
4297
4544
|
|
|
4298
4545
|
local nonce="" total_mismatch=0 total_unreliable=0 total_evaluated=0 rundir="" n_written=0 aborted=0
|
|
4299
4546
|
# Computed ONCE per batch, not per fixture row - see _eval_emit_result's comment.
|
|
4300
|
-
local batch_cliv="" batch_flowv="" batch_grsha=""
|
|
4547
|
+
local batch_cliv="" batch_flowv="" batch_grsha="" prompt_sha_acc="" prompt_sha_enforced=0
|
|
4301
4548
|
if [ "$replay_mode" -eq 1 ]; then
|
|
4302
4549
|
# Replay wrap: skip probe SKIP-exit and _eval_cli_version only. Still compute
|
|
4303
4550
|
# batch_grsha (staleness hard-fail) and _flow_version. Nonce from fixture, never _eval_nonce.
|
|
@@ -4320,12 +4567,17 @@ function cmd_eval {
|
|
|
4320
4567
|
echo " recorded=$rec_sha current=$batch_grsha"
|
|
4321
4568
|
return 1
|
|
4322
4569
|
fi
|
|
4570
|
+
if grep -q '^prompt_sha\.' "$EVAL_REPLAY_DIR/meta" 2>/dev/null; then
|
|
4571
|
+
prompt_sha_enforced=1
|
|
4572
|
+
fi
|
|
4323
4573
|
echo "replay: keyless artifact batch (nonce from fixture, zero live calls)"
|
|
4324
4574
|
else
|
|
4325
|
-
|
|
4575
|
+
_eval_isolated_guard || return 1
|
|
4576
|
+
local probe; probe="$(_eval_isolated_run _eval_probe)"
|
|
4326
4577
|
case "$probe" in
|
|
4327
4578
|
absent)
|
|
4328
|
-
|
|
4579
|
+
_eval_skip_unmeasured "eval"
|
|
4580
|
+
echo "semantic layer unmeasured on this host"
|
|
4329
4581
|
return 0 ;;
|
|
4330
4582
|
fail)
|
|
4331
4583
|
echo "SKIP: 'claude' CLI is present but the sentinel probe did not come back clean (one minimal"
|
|
@@ -4425,13 +4677,36 @@ function cmd_eval {
|
|
|
4425
4677
|
continue
|
|
4426
4678
|
fi
|
|
4427
4679
|
|
|
4680
|
+
local psha
|
|
4681
|
+
psha="$(_eval_prompt_sha "$promptfile")"
|
|
4682
|
+
if [ "$replay_mode" -eq 1 ] && [ "$prompt_sha_enforced" -eq 1 ]; then
|
|
4683
|
+
local rec_psha
|
|
4684
|
+
rec_psha="$(_eval_replay_meta_get "prompt_sha.$fid")" || rec_psha=""
|
|
4685
|
+
if [ -z "$rec_psha" ] || [ "$rec_psha" != "$psha" ]; then
|
|
4686
|
+
echo "eval --replay: fixtures stale — prompt assembly changed; re-record live per ADR re-baseline rule"
|
|
4687
|
+
echo " recorded=$rec_psha current=$psha fixture=$fid"
|
|
4688
|
+
rm -rf "$rundir" 2>/dev/null
|
|
4689
|
+
trap - INT TERM
|
|
4690
|
+
return 1
|
|
4691
|
+
fi
|
|
4692
|
+
fi
|
|
4693
|
+
if [ "$record_mode" -eq 1 ]; then
|
|
4694
|
+
prompt_sha_acc="${prompt_sha_acc}prompt_sha.${fid}=${psha}"$'\n'
|
|
4695
|
+
fi
|
|
4696
|
+
|
|
4428
4697
|
total_evaluated=$((total_evaluated + 1))
|
|
4429
|
-
local flag_count=0 pass_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false
|
|
4698
|
+
local flag_count=0 pass_count=0 invalid_count=0 i=1 model="unknown" retries_sum=0 vote_rate_limited=false vote_timed_out=false
|
|
4430
4699
|
# Filename-safe fixture id for raw-capture paths - manifest fid is only tr -d '\r'-cleaned
|
|
4431
4700
|
# (the v1 trust-boundary is READ-side only, but we are now WRITING keyed by it), so a
|
|
4432
4701
|
# hand-edited or FLOW_EVAL_MANIFEST-overridden manifest cannot traverse out of eval-raw/.
|
|
4433
4702
|
local fid_safe; fid_safe="$(printf '%s' "$fid" | tr -c 'A-Za-z0-9' '-' | sed -E 's/-+/-/g; s/^-//; s/-$//')"
|
|
4434
4703
|
[ -z "$fid_safe" ] && fid_safe="_"
|
|
4704
|
+
if [ "$replay_mode" -eq 0 ]; then
|
|
4705
|
+
local rawdir_prompt="$LOG_DIR/eval-raw/$nonce"
|
|
4706
|
+
if mkdir -p "$rawdir_prompt" 2>/dev/null; then
|
|
4707
|
+
cp "$promptfile" "$rawdir_prompt/${fid_safe}.prompt" 2>/dev/null || true
|
|
4708
|
+
fi
|
|
4709
|
+
fi
|
|
4435
4710
|
while [ "$i" -le "$n" ]; do
|
|
4436
4711
|
local raw rc v raw1="" rc1=0 err1="" err2=""
|
|
4437
4712
|
# Attempt 1 - stderr captured at the OUTER redirection so it never enters the sh -c
|
|
@@ -4442,11 +4717,12 @@ function cmd_eval {
|
|
|
4442
4717
|
raw="$(_eval_replay_vote "$fid" "$i")" || { rm -rf "$rundir" 2>/dev/null; trap - INT TERM; return 1; }
|
|
4443
4718
|
rc=0
|
|
4444
4719
|
else
|
|
4445
|
-
raw="$(_eval_engine_run "$promptfile" "$timeout" 2>"$err1")"; rc=$?
|
|
4720
|
+
raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>"$err1")"; rc=$?
|
|
4446
4721
|
fi
|
|
4447
4722
|
raw1="$raw"; rc1="$rc"
|
|
4448
4723
|
if [ "$rc" -eq 124 ]; then
|
|
4449
4724
|
v=INVALID
|
|
4725
|
+
vote_timed_out=true
|
|
4450
4726
|
else
|
|
4451
4727
|
v="$(_eval_parse_verdict "$raw" "$nonce")"
|
|
4452
4728
|
[ "$model" = "unknown" ] && model="$(_eval_parse_model "$raw")"
|
|
@@ -4477,10 +4753,10 @@ function cmd_eval {
|
|
|
4477
4753
|
raw="$(_eval_replay_vote "$fid" "$i")" || { rm -rf "$rundir" 2>/dev/null; trap - INT TERM; return 1; }
|
|
4478
4754
|
rc=0
|
|
4479
4755
|
else
|
|
4480
|
-
raw="$(_eval_engine_run "$promptfile" "$timeout" 2>"$err2")"; rc=$?
|
|
4756
|
+
raw="$(_eval_isolated_run _eval_engine_run "$promptfile" "$timeout" 2>"$err2")"; rc=$?
|
|
4481
4757
|
fi
|
|
4482
4758
|
retries_sum=$((retries_sum + 1))
|
|
4483
|
-
if [ "$rc" -eq 124 ]; then v=INVALID; else v="$(_eval_parse_verdict "$raw" "$nonce")"; fi
|
|
4759
|
+
if [ "$rc" -eq 124 ]; then v=INVALID; vote_timed_out=true; else v="$(_eval_parse_verdict "$raw" "$nonce")"; fi
|
|
4484
4760
|
[ "$model" = "unknown" ] && [ "$rc" -ne 124 ] && model="$(_eval_parse_model "$raw")"
|
|
4485
4761
|
local rl2; rl2="$(_eval_parse_rate_limited "$raw")"
|
|
4486
4762
|
[ "$rl2" = "true" ] && vote_rate_limited=true
|
|
@@ -4555,7 +4831,7 @@ function cmd_eval {
|
|
|
4555
4831
|
fi
|
|
4556
4832
|
|
|
4557
4833
|
if [ "$replay_mode" -eq 0 ]; then
|
|
4558
|
-
_eval_emit_result "$nonce" "$fid" "$fstage" "$fexpected" "$verdict" "$match" "$flag_count" "$pass_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_grsha" "$retries_sum" "$vote_rate_limited"
|
|
4834
|
+
_eval_emit_result "$nonce" "$fid" "$fstage" "$fexpected" "$verdict" "$match" "$flag_count" "$pass_count" "$invalid_count" "$n" "$model" "$batch_cliv" "$batch_flowv" "$batch_grsha" "$retries_sum" "$vote_rate_limited" "$psha" "$vote_timed_out"
|
|
4559
4835
|
fi
|
|
4560
4836
|
n_written=$((n_written + 1))
|
|
4561
4837
|
|
|
@@ -4587,7 +4863,7 @@ function cmd_eval {
|
|
|
4587
4863
|
_eval_emit_batch_marker "$nonce" done "$n_written"
|
|
4588
4864
|
fi
|
|
4589
4865
|
if [ "$record_mode" -eq 1 ] && [ "$aborted" -eq 0 ]; then
|
|
4590
|
-
_eval_replay_write_meta "$nonce" "$batch_grsha" "${model:-unknown}" "$n"
|
|
4866
|
+
_eval_replay_write_meta "$nonce" "$batch_grsha" "${model:-unknown}" "$n" "$prompt_sha_acc"
|
|
4591
4867
|
fi
|
|
4592
4868
|
|
|
4593
4869
|
echo
|