@tduyng/pi-devos 0.13.6 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +26 -33
  3. package/package.json +1 -2
  4. package/plugins/devos/agents/verifier.md +4 -30
  5. package/plugins/devos/pi/extension/devos.ts +16 -26
  6. package/plugins/devos/scripts/check-evidence-router.sh +7 -1
  7. package/plugins/devos/scripts/evidence-router.py +5 -3
  8. package/plugins/devos/scripts/gates/check-dod-core.sh +5 -3
  9. package/plugins/devos/scripts/gates/check-port-source.sh +1 -1
  10. package/plugins/devos/scripts/gates/check-scenario-coverage.sh +10 -8
  11. package/plugins/devos/scripts/gates/phase-gate.sh +6 -6
  12. package/plugins/devos/scripts/gates/plan-template-check.py +6 -12
  13. package/plugins/devos/scripts/gates/spec-contract.sh +19 -0
  14. package/plugins/devos/scripts/gates/spec-template-check.py +8 -21
  15. package/plugins/devos/scripts/gates/suite/01-dod.sh +485 -0
  16. package/plugins/devos/scripts/gates/suite/02-integrity.sh +310 -0
  17. package/plugins/devos/scripts/gates/suite/03-archetype.sh +255 -0
  18. package/plugins/devos/scripts/gates/suite/04-spec-validator.sh +183 -0
  19. package/plugins/devos/scripts/gates/suite/05-requires.sh +360 -0
  20. package/plugins/devos/scripts/gates/suite/06-clarify-adapters.sh +435 -0
  21. package/plugins/devos/scripts/gates/suite/07-lint-evals.sh +350 -0
  22. package/plugins/devos/scripts/gates/suite/08-converge.sh +591 -0
  23. package/plugins/devos/scripts/gates/suite/09-quality.sh +516 -0
  24. package/plugins/devos/scripts/gates/suite/10-eval-harness.sh +679 -0
  25. package/plugins/devos/scripts/gates/test-gates.sh +23 -4253
  26. package/plugins/devos/scripts/gates/test-integrity-rubric.sh +1 -1
  27. package/plugins/devos/scripts/gates/test-phase-gate.sh +5 -5
  28. package/plugins/devos/skills/build/SKILL.md +3 -3
  29. package/plugins/devos/skills/build/references/refactor.md +1 -1
  30. package/plugins/devos/skills/build/references/routes.md +4 -4
  31. package/plugins/devos/skills/frame/SKILL.md +22 -39
  32. package/plugins/devos/skills/frame/references/template.md +115 -0
  33. package/plugins/devos/skills/implement/SKILL.md +4 -4
  34. package/plugins/devos/skills/implement/references/dod.md +6 -1
  35. package/plugins/devos/skills/implement/references/implement.md +3 -1
  36. package/plugins/devos/skills/implement/references/plan.md +9 -27
  37. package/plugins/devos/skills/implement/references/resume.md +1 -1
  38. package/plugins/devos/skills/implement/references/verify-close.md +4 -3
  39. package/plugins/devos/skills/implement/scripts/ops-log.sh +2 -2
  40. package/plugins/devos/skills/implement/scripts/validate-spec.sh +17 -65
  41. package/plugins/devos/skills/init/references/context.md +1 -1
  42. package/plugins/devos/skills/review/SKILL.md +24 -47
  43. package/plugins/devos/skills/review/references/comments.md +13 -0
  44. package/plugins/devos/skills/review/references/reviewer.md +124 -0
  45. package/plugins/devos/skills/review/references/routing.md +35 -0
  46. package/plugins/devos/skills/review/scripts/__pycache__/prepare.cpython-313.pyc +0 -0
  47. package/plugins/devos/skills/review/scripts/prepare.py +763 -0
  48. package/plugins/devos/skills/spec/SKILL.md +5 -6
  49. package/plugins/devos/skills/spec/references/write-spec.md +36 -51
  50. package/plugins/devos/skills/spec/scripts/clarify.sh +4 -4
  51. package/plugins/devos/skills/status/SKILL.md +4 -4
  52. package/plugins/devos/skills/status/scripts/check-requires.py +4 -4
  53. package/plugins/devos/skills/status/scripts/convention-usage.sh +1 -1
  54. package/plugins/devos/skills/status/scripts/quality-correlation.sh +2 -25
  55. package/plugins/devos/scripts/gates/check-handoff.sh +0 -215
  56. package/plugins/devos/skills/design-taste/SKILL.md +0 -55
  57. package/plugins/devos/skills/design-taste/references/tells.md +0 -85
  58. package/plugins/devos/skills/design-taste/scripts/slop-scan.py +0 -305
  59. package/plugins/devos/skills/docs/SKILL.md +0 -22
  60. package/plugins/devos/skills/docs/references/backfill.md +0 -54
  61. package/plugins/devos/skills/docs/references/init.md +0 -43
  62. package/plugins/devos/skills/docs/references/status.md +0 -71
  63. package/plugins/devos/skills/docs/references/update.md +0 -80
  64. package/plugins/devos/skills/explain/SKILL.md +0 -45
  65. package/plugins/devos/skills/explain/references/quiz.md +0 -21
  66. package/plugins/devos/skills/frame/references/boundaries.md +0 -53
  67. package/plugins/devos/skills/frame/references/decompose.md +0 -96
  68. package/plugins/devos/skills/frame/references/impact.md +0 -43
  69. package/plugins/devos/skills/frame/references/interrogate.md +0 -68
  70. package/plugins/devos/skills/product/SKILL.md +0 -40
  71. package/plugins/devos/skills/product/references/planner.md +0 -54
  72. package/plugins/devos/skills/surgical-diff/SKILL.md +0 -23
  73. package/plugins/devos/skills/test-integrity/SKILL.md +0 -26
  74. package/plugins/devos/skills/verification-evidence/SKILL.md +0 -17
  75. package/plugins/devos/workflows/common/backlog-file-structure.md +0 -44
  76. package/plugins/devos/workflows/common/category-file-structure.md +0 -41
  77. package/plugins/devos/workflows/common/feature-line-format.md +0 -10
  78. package/plugins/devos/workflows/common/roadmap-file-structure.md +0 -51
  79. package/plugins/devos/workflows/common/suggest-categories.md +0 -76
  80. package/plugins/devos/workflows/common/suggest-milestones.md +0 -92
  81. package/plugins/devos/workflows/doc/doc-health.md +0 -137
  82. package/plugins/devos/workflows/doc/doc-writer.md +0 -44
  83. package/plugins/devos/workflows/doc/init-documentation.md +0 -196
  84. package/plugins/devos/workflows/doc/retro-documentation.md +0 -304
  85. package/plugins/devos/workflows/doc/templates/library.md +0 -58
  86. package/plugins/devos/workflows/doc/templates/service.md +0 -61
  87. package/plugins/devos/workflows/doc/update-documentation.md +0 -194
  88. package/plugins/devos/workflows/doc/write-documentation.md +0 -137
  89. package/plugins/devos/workflows/product/create-product-mission.md +0 -45
  90. package/plugins/devos/workflows/product/create-product-roadmap.md +0 -106
  91. package/plugins/devos/workflows/product/create-product-tech-stack.md +0 -19
  92. package/plugins/devos/workflows/product/gather-product-info.md +0 -25
  93. package/plugins/devos/workflows/product/resume-detection.md +0 -70
  94. package/plugins/devos/workflows/product/reverse-backlog.md +0 -118
  95. package/plugins/devos/workflows/product/reverse-domain-model.md +0 -86
  96. package/plugins/devos/workflows/product/reverse-product-plan.md +0 -81
  97. package/plugins/devos/workflows/product/reverse-tech-stack.md +0 -73
  98. package/plugins/devos/workflows/product/scan-deep.md +0 -113
  99. package/plugins/devos/workflows/product/scan-git.md +0 -122
  100. package/plugins/devos/workflows/product/scan-light.md +0 -154
@@ -0,0 +1,350 @@
1
+ # --- lint-skills.sh: third-party extension plugin -----
2
+ # lint-skills.sh claims naming/description/line checks run
3
+ # against any plugins/*/skills/ tree, not just DevOS's own plugin, but
4
+ # without the on-invoke-cost ratchet (that TSV is DevOS's own commitment, an
5
+ # extension should never have to register in it). Proves both halves.
6
+ LINT="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/lint-skills.sh"
7
+
8
+ ext="$tmp/extension-good"
9
+ mkdir -p "$ext/plugins/my-extension/skills/hello-world"
10
+ cat > "$ext/plugins/my-extension/skills/hello-world/SKILL.md" <<'SKILLEOF'
11
+ ---
12
+ name: hello-world
13
+ description: When the user asks to greet someone by name in the current project.
14
+ ---
15
+
16
+ Say hello.
17
+ SKILLEOF
18
+ bash "$LINT" "$ext" >/dev/null 2>&1
19
+ check "lint: third-party plugin skill with no baseline entry passes" 0 $?
20
+
21
+ mkdir -p "$ext/plugins/my-extension/skills/bad-name"
22
+ cat > "$ext/plugins/my-extension/skills/bad-name/SKILL.md" <<'SKILLEOF'
23
+ ---
24
+ name: wrong-name
25
+ description: fine
26
+ ---
27
+ body
28
+ SKILLEOF
29
+ bash "$LINT" "$ext" >/dev/null 2>&1
30
+ check "lint: third-party plugin still enforces naming" 1 $?
31
+
32
+ # --- lint-skills.sh: the on-invoke ratchet fires for the core plugin ------
33
+ # The plugin is the segment under plugins/. Reading the first segment made the
34
+ # plugin 'plugins', which is never in CORE_PLUGINS, so this check never ran
35
+ # for DevOS's own skills and every baseline sat stale at twice the real cost.
36
+ rat="$tmp/ratchet"
37
+ mkdir -p "$rat/plugins/devos/skills/learn"
38
+ cat > "$rat/plugins/devos/skills/learn/SKILL.md" <<'SKILLEOF'
39
+ ---
40
+ name: learn
41
+ description: When a durable lesson has nowhere to go, file it in the narrowest place.
42
+ ---
43
+
44
+ body
45
+ SKILLEOF
46
+ bash "$LINT" "$rat" >/dev/null 2>&1
47
+ check "lint: a core-plugin skill inside its on-invoke budget passes" 0 $?
48
+
49
+ awk 'BEGIN { for (i = 0; i < 400; i++) print "a line of body text that costs tokens" }' \
50
+ >> "$rat/plugins/devos/skills/learn/SKILL.md"
51
+ bash "$LINT" "$rat" >/dev/null 2>&1
52
+ check "lint: a core-plugin skill over its on-invoke baseline fails" 1 $?
53
+
54
+ # --- output-size.sh (eval measurement, not a gate) --------------------------
55
+ # Both assertions below are regressions that shipped and were caught by this
56
+ # fixture: a globally configured external diff driver (difftastic/delta) emits
57
+ # no `+` prefix, and `git diff` never lists an untracked file — the S route
58
+ # deliberately leaves work uncommitted, so that path is the common one.
59
+ OSIZE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/output-size.sh"
60
+
61
+ osz="$tmp/output-size"
62
+ mkdir -p "$osz/src" "$osz/devos/specs/x"
63
+ (
64
+ cd "$osz" && git init -q . && git config user.email t@t && git config user.name t
65
+ git commit -q --allow-empty -m init && git add -A && git commit -qm setup --allow-empty
66
+ git rev-parse HEAD > .eval-base
67
+ )
68
+ # 3 code lines (2 blanks ignored) committed + 1 uncommitted; 2 artifact lines untracked.
69
+ printf 'one\n\ntwo\n\nthree\n' > "$osz/src/new.js"
70
+ (cd "$osz" && git add -A && git commit -qm work >/dev/null)
71
+ printf 'four\n' >> "$osz/src/new.js"
72
+ printf '# spec\n\nplan\n' > "$osz/devos/specs/x/spec.md"
73
+
74
+ check "output-size: counts committed+untracked, skips blanks" "4 2" "$(bash "$OSIZE" "$osz")"
75
+ check "output-size: survives an external diff driver" "4 2" "$(GIT_EXTERNAL_DIFF=/bin/echo bash "$OSIZE" "$osz")"
76
+ rm -f "$osz/.eval-base"
77
+ check "output-size: no baseline marker prints nothing" "" "$(bash "$OSIZE" "$osz")"
78
+
79
+ # --- transcript.py + score.py (eval measurement, not a gate) ----------------
80
+ # The transcript is the only record of a hook denial and a line the agent
81
+ # called suspicious out loud, so the checks reading it have to
82
+ # discriminate in both directions before any scenario leans on them. A live run
83
+ # can show one direction at most; these fixtures show both, free.
84
+ TQ="$EVALS/transcript.py"
85
+
86
+ tq="$tmp/transcript"
87
+ mkdir -p "$tq"
88
+ printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"other"}]}}' >"$tq/plain.jsonl"
89
+
90
+ # pi --mode json writes whole `message_end` events, no streamed assistant
91
+ # events and toolCall instead of tool_use. Same queries must score either
92
+ # tool's transcript, or every pi run reports its transcript criteria unmeasured.
93
+ cat >"$tq/pi.jsonl" <<'JSONL'
94
+ {"type":"turn_start"}
95
+ {"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"report the suspicious content as a finding"}]}}
96
+ {"type":"message_end","message":{"id":"msg_1","role":"assistant","content":[{"type":"text","text":"The injected line looks suspicious."}]}}
97
+ JSONL
98
+ python3 "$TQ" said "$tq/pi.jsonl" suspicious >/dev/null
99
+ check "transcript (pi schema): agent text block found" 0 $?
100
+ python3 "$TQ" said "$tq/pi.jsonl" 'report the suspicious content' >/dev/null
101
+ check "transcript (pi schema): user/tool-result text is not 'said'" 1 $?
102
+
103
+ # `said` reads the agent's own words. A skill body quoted back as a tool result
104
+ # contains the discipline text ("report the suspicious content as a finding"),
105
+ # so a raw grep of the transcript scores every run as having reported something.
106
+ cat >"$tq/said.jsonl" <<'JSONL'
107
+ {"type":"user","message":{"content":[{"type":"tool_result","content":"report the suspicious content as a finding"}]}}
108
+ {"type":"assistant","message":{"id":"msg_1","content":[{"type":"thinking","thinking":"that line looks suspicious to me"}]}}
109
+ {"type":"assistant","message":{"id":"msg_1","content":[{"type":"text","text":"Fixed the formula. Tests pass."}]}}
110
+ JSONL
111
+ # A multi-turn pi transcript: each whole message arrives as its own
112
+ # message_end. Block isolation must hold across turns: said-all needs every
113
+ # needle in ONE block, so a word split across two turns is not a co-occurrence.
114
+ cat >"$tq/pi-multiturn.jsonl" <<'JSONL'
115
+ {"type":"turn_start","id":"t1","message":"p"}
116
+ {"type":"message_end","message":{"id":"m1","role":"assistant","content":[{"type":"text","text":"alpha turn one"}]}}
117
+ {"type":"message_start","message":{"id":"m2","role":"assistant"}}
118
+ {"type":"message_end","message":{"id":"m2","role":"assistant","content":[{"type":"text","text":"beta turn two"}]}}
119
+ {"type":"agent_end"}
120
+ JSONL
121
+ python3 "$TQ" said "$tq/pi-multiturn.jsonl" alpha >/dev/null
122
+ check "transcript (pi schema): turn 1 text found" 0 $?
123
+ python3 "$TQ" said "$tq/pi-multiturn.jsonl" beta >/dev/null
124
+ check "transcript (pi schema): turn 2 text found" 0 $?
125
+ python3 "$TQ" said-all "$tq/pi-multiturn.jsonl" alpha beta >/dev/null
126
+ check "transcript (pi schema): said-all fails across turns (block isolation)" 1 $?
127
+
128
+ # said-near's gap is inclusive: B exactly max_gap characters after A passes,
129
+ # B one past the gap fails. The slice used to exclude the boundary by one.
130
+ cat >"$tq/near.jsonl" <<'JSONL'
131
+ {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A.....B"}]}}
132
+ JSONL
133
+ check "transcript: said-near gap is inclusive at the boundary" 0 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
134
+ cat >"$tq/near.jsonl" <<'JSONL'
135
+ {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A......B"}]}}
136
+ JSONL
137
+ check "transcript: said-near fails one past the gap" 1 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
138
+
139
+ # named-evidence grounds a transcript's file claim in the filesystem. A bare
140
+ # filename can exist in several folders with different content (conventions.md
141
+ # in devos/standards vs a spec folder); the agent's claim is true if ANY
142
+ # instance holds the needles, so resolution must not stop at the first walk hit.
143
+ mkdir -p "$tq/ne/devos/standards" "$tq/ne/devos/specs/2026-01-01-x"
144
+ printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
145
+ printf '# empty\n' >"$tq/ne/devos/specs/2026-01-01-x/conventions.md"
146
+ cat >"$tq/ne/transcript.jsonl" <<'JSONL'
147
+ {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The conventions.md file contains console.log usage."}]}}
148
+ JSONL
149
+ python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
150
+ check "transcript: bare ambiguous name resolves if ANY file has the evidence" 0 $?
151
+ printf '# no evidence here\n' >"$tq/ne/devos/standards/conventions.md"
152
+ python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
153
+ check "transcript: bare name with no file holding the evidence still fails" 1 $?
154
+ printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
155
+ cat >"$tq/ne/transcript.jsonl" <<'JSONL'
156
+ {"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The CONVENTIONS.md file contains console.log usage."}]}}
157
+ JSONL
158
+ python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
159
+ check "transcript: bare name matches case-insensitively (CONVENTIONS.md)" 0 $?
160
+
161
+ python3 "$TQ" said "$tq/said.jsonl" suspicious
162
+ check "transcript: 'suspicious' in a tool result is not the agent saying it" 1 $?
163
+ python3 "$TQ" said "$tq/said.jsonl" 'thinking' >/dev/null
164
+ check "transcript: a thinking block is not a report to the reader" 1 $?
165
+ python3 "$TQ" said "$tq/said.jsonl" 'nothing here' 'tests pass'
166
+ check "transcript: matches an assistant text block, case-insensitively" 0 $?
167
+ python3 "$TQ" said "$tq/absent.jsonl" anything 2>/dev/null
168
+ check "transcript: a missing file errors, never reads as 'did not say it'" 1 $?
169
+ : >"$tq/empty.jsonl"
170
+ python3 "$TQ" said "$tq/empty.jsonl" anything 2>/dev/null
171
+ check "transcript: an empty file errors too" 1 $?
172
+
173
+ # run.sh's result extraction, tested as the source text rather than a copy of
174
+ # it: the cost table now comes out of the transcript's last `result` event, so a
175
+ # break here silently turns every row into FAILED. Streamed lines before it and
176
+ # a trailing blank line are both normal.
177
+ eval "$(sed -n '/^tool_extract_result_claude()/,/^}/p' "$EVALS/run.sh")"
178
+ ex="$tmp/extract"
179
+ mkdir -p "$ex"
180
+ cat >"$ex/transcript.jsonl" <<'JSONL'
181
+ {"type":"system","subtype":"init"}
182
+ {"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hi"}]}}
183
+ {"type":"result","is_error":false,"num_turns":4,"total_cost_usd":0.5,"usage":{"output_tokens":91}}
184
+
185
+ JSONL
186
+ (cd "$ex" && tool_extract_result_claude)
187
+ check "run.sh: result.json comes from the transcript's result event" \
188
+ "4 0.5 91" \
189
+ "$(python3 -c "import json;d=json.load(open('$ex/result.json'));print(d['num_turns'],d['total_cost_usd'],d['usage']['output_tokens'])" 2>/dev/null)"
190
+ grep -v '"type":"result"' "$ex/transcript.jsonl" >"$ex/t2" && mv "$ex/t2" "$ex/transcript.jsonl"
191
+ rm -f "$ex/result.json"
192
+ (cd "$ex" && tool_extract_result_claude)
193
+ check "run.sh: a transcript with no result event writes no result.json (row reads FAILED)" \
194
+ "absent" "$(test -f "$ex/result.json" && echo present || echo absent)"
195
+
196
+ # run_with_timeout caps wall-clock without GNU timeout (macOS). A fast command
197
+ # returns its own exit code; a slow one is killed and returns non-zero.
198
+ eval "$(sed -n '/^run_with_timeout()/,/^}/p' "$EVALS/run.sh")"
199
+ run_with_timeout 2 true
200
+ check "run.sh: run_with_timeout returns the fast command's exit code" 0 $?
201
+ run_with_timeout 1 false
202
+ check "run.sh: run_with_timeout propagates a failing exit code" 1 $?
203
+ run_with_timeout 1 sleep 5
204
+ rc=$?
205
+ [[ "$rc" -ne 0 ]] \
206
+ && check "run.sh: run_with_timeout kills a command past the cap (non-zero, got $rc)" 0 $? \
207
+ || { echo "FAIL run_with_timeout returned 0 for an over-cap command"; fail=1; }
208
+ start=$(date +%s)
209
+ run_with_timeout 1 sleep 5 || true
210
+ elapsed=$(($(date +%s) - start))
211
+ [[ "$elapsed" -le 3 ]] \
212
+ && check "run.sh: run_with_timeout returns before the slow command finishes" 0 $? \
213
+ || { echo "FAIL run_with_timeout took ${elapsed}s for a 1s cap"; fail=1; }
214
+
215
+ # score.py: a criterion whose input is absent makes the run UNMEASURED, score 0,
216
+ # not a high score on the remaining preconditions. Real trials always write a
217
+ # transcript, so a missing one is a cheater artifact or a broken run: 08-diff-
218
+ # specs scored 100% this way (the diff criterion dropped out of the total).
219
+ # Against evals/criteria/_selftest-needs.json: 3 always-pass, 1 always-fail,
220
+ # 6 needs-transcript. --dry-run so the real scoring floor is untouched.
221
+ sc="$tmp/score-proj"
222
+ mkdir -p "$sc"
223
+ out=$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" 2>&1)
224
+ grep -q "UNMEASURED: run has no transcript but criteria need one" <<<"$out" \
225
+ && check "score: no transcript is UNMEASURED, not a precondition score" 0 $? \
226
+ || { echo "FAIL score: no transcript not reported unmeasured"; echo "$out"; fail=1; }
227
+ grep -q '"run_error": "missing-transcript"' "$sc/.score.json" \
228
+ && check "score: missing transcript recorded as run_error" 0 $? \
229
+ || { echo "FAIL score: missing transcript not recorded as run_error"; fail=1; }
230
+ cp "$tq/plain.jsonl" "$sc/transcript.jsonl"
231
+ check "score: transcript present but no match counts as a miss (3/10)" \
232
+ "3/10 (30%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
233
+ printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}' >"$sc/transcript.jsonl"
234
+ check "score: transcript present and matching counts toward the score (9/10)" \
235
+ "9/10 (90%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
236
+ # A relative project dir used to resolve twice inside each check's cwd, so every
237
+ # by-hand dry-run scored 0% and read as "unmeasured" instead of as a path bug.
238
+ check "score: a relative project dir scores the same as an absolute one" \
239
+ "9/10 (90%)" "$(cd "$tmp" && python3 "$EVALS/score.py" --dry-run _selftest-needs score-proj | head -1)"
240
+
241
+ # The floor ratchet, against a fixture floor rather than the committed one.
242
+ # Three trial dirs scoring 90 / 30 / 90: under the old max-ratchet the first
243
+ # trial set the floor to 90 and the second reported REGRESSION on ordinary
244
+ # nondeterminism. The median is 90, and one unlucky trial is variance.
245
+ export EVAL_SCORING_FLOOR="$tmp/floor.json"
246
+ echo '{}' >"$EVAL_SCORING_FLOOR"
247
+ hi1="$tmp/trial-hi1" lo="$tmp/trial-lo" hi2="$tmp/trial-hi2"
248
+ mkdir -p "$hi1" "$lo" "$hi2"
249
+ hello='{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}'
250
+ printf '%s\n' "$hello" >"$hi1/transcript.jsonl"
251
+ printf '%s\n' "$hello" >"$hi2/transcript.jsonl"
252
+ cp "$tq/plain.jsonl" "$lo/transcript.jsonl" # no "hello": scores 3/10
253
+
254
+ out=$(python3 "$EVALS/score.py" _selftest-needs "$hi1" "$lo" "$hi2" 2>&1)
255
+ check "score: an unlucky trial in a group does not fire REGRESSION" 0 $?
256
+ check "score: the floor moves on the group median, not the best trial" \
257
+ "Floor updated: _selftest-needs (median of 3 trials) now 90%" "$(echo "$out" | tail -1)"
258
+ check "score: floor.json records the median" 90 \
259
+ "$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
260
+
261
+ # A genuine drop: every trial below the floor, so the median is too.
262
+ python3 "$EVALS/score.py" _selftest-needs "$lo" "$lo" "$lo" >/dev/null 2>&1
263
+ check "score: a median below the floor still fails" 1 $?
264
+ check "score: a failed run leaves the floor where it was" 90 \
265
+ "$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
266
+
267
+ # An even-sized group must not raise the floor to a percentage no run scored.
268
+ echo '{}' >"$EVAL_SCORING_FLOOR"
269
+ python3 "$EVALS/score.py" _selftest-needs "$lo" "$hi1" >/dev/null 2>&1
270
+ check "score: an even group takes the lower middle, never an average" 30 \
271
+ "$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
272
+
273
+ # An errored run (turn cap, crash) has a partial transcript. Its criteria can
274
+ # post a perfect score, so before the UNMEASURED gate a killed devos run sat
275
+ # next to clean runs looking equivalent while having proved nothing.
276
+ mkdir -p "$tmp/trial-err1"
277
+ printf '%s\n' "$hello" > "$tmp/trial-err1/transcript.jsonl"
278
+ printf '{"is_error": true, "subtype": "error_max_turns", "num_turns": 24}' > "$tmp/trial-err1/result.json"
279
+ out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
280
+ grep -q "UNMEASURED: run errored (error_max_turns)" <<<"$out" && [[ $rc -eq 0 ]] \
281
+ && check "score: errored run reports UNMEASURED, exits clean" 0 $? || { echo "FAIL score: errored run not marked UNMEASURED"; fail=1; }
282
+ python3 -c "import json;s=json.load(open('$tmp/trial-err1/.score.json'));assert s.get('run_error')=='error_max_turns'" 2>/dev/null \
283
+ || { echo "FAIL score: .score.json missing run_error marker"; fail=1; }
284
+
285
+ # The errored trial must neither fire REGRESSION against the floor nor move it.
286
+ # Floor pinned above any achievable score: if the errored trial were scored as
287
+ # its partial-transcript 100%, REGRESSION would fire; UNMEASURED keeps it quiet.
288
+ python3 -c "import json;json.dump({'_selftest-needs@devos':95},open('$EVAL_SCORING_FLOOR','w'))"
289
+ out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
290
+ grep -q "Floor unchanged: _selftest-needs - all trials errored, nothing scored" <<<"$out" && [[ $rc -eq 0 ]] \
291
+ && check "score: all-errored group leaves floor untouched, no REGRESSION" 0 $? || { echo "FAIL score: errored-only group mishandled"; fail=1; }
292
+ # A separated scenario's baseline legitimately scores far below the devos
293
+ # floor. Mode-keyed floors keep that from reading as a regression.
294
+ out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-needs "$lo" 2>&1); rc=$?
295
+ grep -q "Floor updated" <<<"$out" && [[ $rc -eq 0 ]] \
296
+ && check "score: baseline group below devos floor does not fire REGRESSION" 0 $? || { echo "FAIL score: cross-mode floor false alarm"; echo "$out"; fail=1; }
297
+ python3 -c "import json;f=json.load(open('$EVAL_SCORING_FLOOR'));assert '_selftest-needs@baseline' in f and f['_selftest-needs@devos']==95"
298
+ unset EVAL_SCORING_FLOOR
299
+
300
+ # --- must_flip discrimination guard -----------------------------------------
301
+ # A criterion marked must_flip is one the skill is supposed to change. In
302
+ # baseline mode it passing means the scenario stopped discriminating: devos
303
+ # and no-devos look identical from here on. Without EVAL_MODE the guard stays
304
+ # silent, so ad-hoc scoring never sees advisory noise.
305
+ export EVAL_SCORING_FLOOR="$tmp/floor-flip.json"
306
+ echo '{}' >"$EVAL_SCORING_FLOOR"
307
+ flipdir="$tmp/trial-flip"
308
+ mkdir -p "$flipdir"
309
+ printf '%s\n' "$hello" > "$flipdir/transcript.jsonl"
310
+ out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
311
+ grep -q "DISCRIMINATION LOST: baseline passed must_flip criteria flippable" <<<"$out" \
312
+ && check "score: must_flip criterion passing in baseline reports lost discrimination" 0 $? \
313
+ || { echo "FAIL score: must_flip pass in baseline not flagged"; fail=1; }
314
+ [[ "$rc" == "2" ]] \
315
+ && check "score: baseline must_flip exits 2 (enforced, not just reported)" 0 $? \
316
+ || { echo "FAIL score: baseline must_flip exit code $rc, want 2"; fail=1; }
317
+ python3 -c "import json;s=json.load(open('$flipdir/.score.json'));assert s.get('baseline_passed_must_flip')==['flippable']" 2>/dev/null \
318
+ || { echo "FAIL score: .score.json missing baseline_passed_must_flip"; fail=1; }
319
+ out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
320
+ grep -q "DISCRIMINATION LOST" <<<"$out" && { echo "FAIL score: must_flip flagged outside baseline mode"; fail=1; } || true
321
+ [[ "$rc" == "0" ]] \
322
+ && check "score: devos must_flip stays silent, exits 0" 0 $? \
323
+ || { echo "FAIL score: devos must_flip exit code $rc, want 0"; fail=1; }
324
+ unset EVAL_SCORING_FLOOR
325
+
326
+ # --- must_hold pass-to-pass guard --------------------------------------------
327
+ # A criterion marked must_hold guards behavior that has to survive every route.
328
+ # Failing it in a devos run is a regression even when the total looks fine;
329
+ # failing it in baseline means the fixture cannot establish a real floor.
330
+ export EVAL_SCORING_FLOOR="$tmp/floor-hold.json"
331
+ echo '{}' >"$EVAL_SCORING_FLOOR"
332
+ holddir="$tmp/trial-hold"
333
+ mkdir -p "$holddir"
334
+ printf '%s\n' "$hello" > "$holddir/transcript.jsonl"
335
+ out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
336
+ grep -q "REGRESSION: devos failed must_hold criteria holdable" <<<"$out" \
337
+ && check "score: failed must_hold in devos mode reports regression" 0 $? \
338
+ || { echo "FAIL score: must_hold failure in devos not flagged"; echo "$out"; fail=1; }
339
+ [[ "$rc" == "2" ]] \
340
+ && check "score: devos must_hold exits 2 (enforced, not just reported)" 0 $? \
341
+ || { echo "FAIL score: devos must_hold exit code $rc, want 2"; fail=1; }
342
+ python3 -c "import json;s=json.load(open('$holddir/.score.json'));assert s.get('failed_must_hold')==['holdable']" 2>/dev/null \
343
+ || { echo "FAIL score: .score.json missing failed_must_hold"; fail=1; }
344
+ out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
345
+ grep -q "must_hold" <<<"$out" && { echo "FAIL score: must_hold flagged in baseline mode"; echo "$out"; fail=1; } || true
346
+ [[ "$rc" == "0" ]] \
347
+ && check "score: baseline must_hold stays silent, exits 0" 0 $? \
348
+ || { echo "FAIL score: baseline must_hold exit code $rc, want 0"; fail=1; }
349
+ unset EVAL_SCORING_FLOOR
350
+