@tduyng/pi-devos 0.13.6 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +26 -33
- package/package.json +1 -2
- package/plugins/devos/agents/verifier.md +4 -30
- package/plugins/devos/pi/extension/devos.ts +16 -26
- package/plugins/devos/scripts/check-evidence-router.sh +7 -1
- package/plugins/devos/scripts/evidence-router.py +5 -3
- package/plugins/devos/scripts/gates/check-dod-core.sh +5 -3
- package/plugins/devos/scripts/gates/check-port-source.sh +1 -1
- package/plugins/devos/scripts/gates/check-scenario-coverage.sh +10 -8
- package/plugins/devos/scripts/gates/phase-gate.sh +6 -6
- package/plugins/devos/scripts/gates/plan-template-check.py +6 -12
- package/plugins/devos/scripts/gates/spec-contract.sh +19 -0
- package/plugins/devos/scripts/gates/spec-template-check.py +8 -21
- package/plugins/devos/scripts/gates/suite/01-dod.sh +485 -0
- package/plugins/devos/scripts/gates/suite/02-integrity.sh +310 -0
- package/plugins/devos/scripts/gates/suite/03-archetype.sh +255 -0
- package/plugins/devos/scripts/gates/suite/04-spec-validator.sh +183 -0
- package/plugins/devos/scripts/gates/suite/05-requires.sh +360 -0
- package/plugins/devos/scripts/gates/suite/06-clarify-adapters.sh +435 -0
- package/plugins/devos/scripts/gates/suite/07-lint-evals.sh +350 -0
- package/plugins/devos/scripts/gates/suite/08-converge.sh +591 -0
- package/plugins/devos/scripts/gates/suite/09-quality.sh +516 -0
- package/plugins/devos/scripts/gates/suite/10-eval-harness.sh +679 -0
- package/plugins/devos/scripts/gates/test-gates.sh +23 -4253
- package/plugins/devos/scripts/gates/test-integrity-rubric.sh +1 -1
- package/plugins/devos/scripts/gates/test-phase-gate.sh +5 -5
- package/plugins/devos/skills/build/SKILL.md +3 -3
- package/plugins/devos/skills/build/references/refactor.md +1 -1
- package/plugins/devos/skills/build/references/routes.md +4 -4
- package/plugins/devos/skills/frame/SKILL.md +22 -39
- package/plugins/devos/skills/frame/references/template.md +115 -0
- package/plugins/devos/skills/implement/SKILL.md +4 -4
- package/plugins/devos/skills/implement/references/dod.md +6 -1
- package/plugins/devos/skills/implement/references/implement.md +3 -1
- package/plugins/devos/skills/implement/references/plan.md +9 -27
- package/plugins/devos/skills/implement/references/resume.md +1 -1
- package/plugins/devos/skills/implement/references/verify-close.md +4 -3
- package/plugins/devos/skills/implement/scripts/ops-log.sh +2 -2
- package/plugins/devos/skills/implement/scripts/validate-spec.sh +17 -65
- package/plugins/devos/skills/init/references/context.md +1 -1
- package/plugins/devos/skills/review/SKILL.md +24 -47
- package/plugins/devos/skills/review/references/comments.md +13 -0
- package/plugins/devos/skills/review/references/reviewer.md +124 -0
- package/plugins/devos/skills/review/references/routing.md +35 -0
- package/plugins/devos/skills/review/scripts/__pycache__/prepare.cpython-313.pyc +0 -0
- package/plugins/devos/skills/review/scripts/prepare.py +763 -0
- package/plugins/devos/skills/spec/SKILL.md +5 -6
- package/plugins/devos/skills/spec/references/write-spec.md +36 -51
- package/plugins/devos/skills/spec/scripts/clarify.sh +4 -4
- package/plugins/devos/skills/status/SKILL.md +4 -4
- package/plugins/devos/skills/status/scripts/check-requires.py +4 -4
- package/plugins/devos/skills/status/scripts/convention-usage.sh +1 -1
- package/plugins/devos/skills/status/scripts/quality-correlation.sh +2 -25
- package/plugins/devos/scripts/gates/check-handoff.sh +0 -215
- package/plugins/devos/skills/design-taste/SKILL.md +0 -55
- package/plugins/devos/skills/design-taste/references/tells.md +0 -85
- package/plugins/devos/skills/design-taste/scripts/slop-scan.py +0 -305
- package/plugins/devos/skills/docs/SKILL.md +0 -22
- package/plugins/devos/skills/docs/references/backfill.md +0 -54
- package/plugins/devos/skills/docs/references/init.md +0 -43
- package/plugins/devos/skills/docs/references/status.md +0 -71
- package/plugins/devos/skills/docs/references/update.md +0 -80
- package/plugins/devos/skills/explain/SKILL.md +0 -45
- package/plugins/devos/skills/explain/references/quiz.md +0 -21
- package/plugins/devos/skills/frame/references/boundaries.md +0 -53
- package/plugins/devos/skills/frame/references/decompose.md +0 -96
- package/plugins/devos/skills/frame/references/impact.md +0 -43
- package/plugins/devos/skills/frame/references/interrogate.md +0 -68
- package/plugins/devos/skills/product/SKILL.md +0 -40
- package/plugins/devos/skills/product/references/planner.md +0 -54
- package/plugins/devos/skills/surgical-diff/SKILL.md +0 -23
- package/plugins/devos/skills/test-integrity/SKILL.md +0 -26
- package/plugins/devos/skills/verification-evidence/SKILL.md +0 -17
- package/plugins/devos/workflows/common/backlog-file-structure.md +0 -44
- package/plugins/devos/workflows/common/category-file-structure.md +0 -41
- package/plugins/devos/workflows/common/feature-line-format.md +0 -10
- package/plugins/devos/workflows/common/roadmap-file-structure.md +0 -51
- package/plugins/devos/workflows/common/suggest-categories.md +0 -76
- package/plugins/devos/workflows/common/suggest-milestones.md +0 -92
- package/plugins/devos/workflows/doc/doc-health.md +0 -137
- package/plugins/devos/workflows/doc/doc-writer.md +0 -44
- package/plugins/devos/workflows/doc/init-documentation.md +0 -196
- package/plugins/devos/workflows/doc/retro-documentation.md +0 -304
- package/plugins/devos/workflows/doc/templates/library.md +0 -58
- package/plugins/devos/workflows/doc/templates/service.md +0 -61
- package/plugins/devos/workflows/doc/update-documentation.md +0 -194
- package/plugins/devos/workflows/doc/write-documentation.md +0 -137
- package/plugins/devos/workflows/product/create-product-mission.md +0 -45
- package/plugins/devos/workflows/product/create-product-roadmap.md +0 -106
- package/plugins/devos/workflows/product/create-product-tech-stack.md +0 -19
- package/plugins/devos/workflows/product/gather-product-info.md +0 -25
- package/plugins/devos/workflows/product/resume-detection.md +0 -70
- package/plugins/devos/workflows/product/reverse-backlog.md +0 -118
- package/plugins/devos/workflows/product/reverse-domain-model.md +0 -86
- package/plugins/devos/workflows/product/reverse-product-plan.md +0 -81
- package/plugins/devos/workflows/product/reverse-tech-stack.md +0 -73
- package/plugins/devos/workflows/product/scan-deep.md +0 -113
- package/plugins/devos/workflows/product/scan-git.md +0 -122
- package/plugins/devos/workflows/product/scan-light.md +0 -154
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
# --- lint-skills.sh: third-party extension plugin -----
|
|
2
|
+
# lint-skills.sh claims naming/description/line checks run
|
|
3
|
+
# against any plugins/*/skills/ tree, not just DevOS's own plugin, but
|
|
4
|
+
# without the on-invoke-cost ratchet (that TSV is DevOS's own commitment, an
|
|
5
|
+
# extension should never have to register in it). Proves both halves.
|
|
6
|
+
LINT="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/lint-skills.sh"
|
|
7
|
+
|
|
8
|
+
ext="$tmp/extension-good"
|
|
9
|
+
mkdir -p "$ext/plugins/my-extension/skills/hello-world"
|
|
10
|
+
cat > "$ext/plugins/my-extension/skills/hello-world/SKILL.md" <<'SKILLEOF'
|
|
11
|
+
---
|
|
12
|
+
name: hello-world
|
|
13
|
+
description: When the user asks to greet someone by name in the current project.
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
Say hello.
|
|
17
|
+
SKILLEOF
|
|
18
|
+
bash "$LINT" "$ext" >/dev/null 2>&1
|
|
19
|
+
check "lint: third-party plugin skill with no baseline entry passes" 0 $?
|
|
20
|
+
|
|
21
|
+
mkdir -p "$ext/plugins/my-extension/skills/bad-name"
|
|
22
|
+
cat > "$ext/plugins/my-extension/skills/bad-name/SKILL.md" <<'SKILLEOF'
|
|
23
|
+
---
|
|
24
|
+
name: wrong-name
|
|
25
|
+
description: fine
|
|
26
|
+
---
|
|
27
|
+
body
|
|
28
|
+
SKILLEOF
|
|
29
|
+
bash "$LINT" "$ext" >/dev/null 2>&1
|
|
30
|
+
check "lint: third-party plugin still enforces naming" 1 $?
|
|
31
|
+
|
|
32
|
+
# --- lint-skills.sh: the on-invoke ratchet fires for the core plugin ------
|
|
33
|
+
# The plugin is the segment under plugins/. Reading the first segment made the
|
|
34
|
+
# plugin 'plugins', which is never in CORE_PLUGINS, so this check never ran
|
|
35
|
+
# for DevOS's own skills and every baseline sat stale at twice the real cost.
|
|
36
|
+
rat="$tmp/ratchet"
|
|
37
|
+
mkdir -p "$rat/plugins/devos/skills/learn"
|
|
38
|
+
cat > "$rat/plugins/devos/skills/learn/SKILL.md" <<'SKILLEOF'
|
|
39
|
+
---
|
|
40
|
+
name: learn
|
|
41
|
+
description: When a durable lesson has nowhere to go, file it in the narrowest place.
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
body
|
|
45
|
+
SKILLEOF
|
|
46
|
+
bash "$LINT" "$rat" >/dev/null 2>&1
|
|
47
|
+
check "lint: a core-plugin skill inside its on-invoke budget passes" 0 $?
|
|
48
|
+
|
|
49
|
+
awk 'BEGIN { for (i = 0; i < 400; i++) print "a line of body text that costs tokens" }' \
|
|
50
|
+
>> "$rat/plugins/devos/skills/learn/SKILL.md"
|
|
51
|
+
bash "$LINT" "$rat" >/dev/null 2>&1
|
|
52
|
+
check "lint: a core-plugin skill over its on-invoke baseline fails" 1 $?
|
|
53
|
+
|
|
54
|
+
# --- output-size.sh (eval measurement, not a gate) --------------------------
|
|
55
|
+
# Both assertions below are regressions that shipped and were caught by this
|
|
56
|
+
# fixture: a globally configured external diff driver (difftastic/delta) emits
|
|
57
|
+
# no `+` prefix, and `git diff` never lists an untracked file — the S route
|
|
58
|
+
# deliberately leaves work uncommitted, so that path is the common one.
|
|
59
|
+
OSIZE="$(cd "$(dirname "$DOD")/../../../.." && pwd)/evals/output-size.sh"
|
|
60
|
+
|
|
61
|
+
osz="$tmp/output-size"
|
|
62
|
+
mkdir -p "$osz/src" "$osz/devos/specs/x"
|
|
63
|
+
(
|
|
64
|
+
cd "$osz" && git init -q . && git config user.email t@t && git config user.name t
|
|
65
|
+
git commit -q --allow-empty -m init && git add -A && git commit -qm setup --allow-empty
|
|
66
|
+
git rev-parse HEAD > .eval-base
|
|
67
|
+
)
|
|
68
|
+
# 3 code lines (2 blanks ignored) committed + 1 uncommitted; 2 artifact lines untracked.
|
|
69
|
+
printf 'one\n\ntwo\n\nthree\n' > "$osz/src/new.js"
|
|
70
|
+
(cd "$osz" && git add -A && git commit -qm work >/dev/null)
|
|
71
|
+
printf 'four\n' >> "$osz/src/new.js"
|
|
72
|
+
printf '# spec\n\nplan\n' > "$osz/devos/specs/x/spec.md"
|
|
73
|
+
|
|
74
|
+
check "output-size: counts committed+untracked, skips blanks" "4 2" "$(bash "$OSIZE" "$osz")"
|
|
75
|
+
check "output-size: survives an external diff driver" "4 2" "$(GIT_EXTERNAL_DIFF=/bin/echo bash "$OSIZE" "$osz")"
|
|
76
|
+
rm -f "$osz/.eval-base"
|
|
77
|
+
check "output-size: no baseline marker prints nothing" "" "$(bash "$OSIZE" "$osz")"
|
|
78
|
+
|
|
79
|
+
# --- transcript.py + score.py (eval measurement, not a gate) ----------------
|
|
80
|
+
# The transcript is the only record of a hook denial and a line the agent
|
|
81
|
+
# called suspicious out loud, so the checks reading it have to
|
|
82
|
+
# discriminate in both directions before any scenario leans on them. A live run
|
|
83
|
+
# can show one direction at most; these fixtures show both, free.
|
|
84
|
+
TQ="$EVALS/transcript.py"
|
|
85
|
+
|
|
86
|
+
tq="$tmp/transcript"
|
|
87
|
+
mkdir -p "$tq"
|
|
88
|
+
printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"other"}]}}' >"$tq/plain.jsonl"
|
|
89
|
+
|
|
90
|
+
# pi --mode json writes whole `message_end` events, no streamed assistant
|
|
91
|
+
# events and toolCall instead of tool_use. Same queries must score either
|
|
92
|
+
# tool's transcript, or every pi run reports its transcript criteria unmeasured.
|
|
93
|
+
cat >"$tq/pi.jsonl" <<'JSONL'
|
|
94
|
+
{"type":"turn_start"}
|
|
95
|
+
{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"report the suspicious content as a finding"}]}}
|
|
96
|
+
{"type":"message_end","message":{"id":"msg_1","role":"assistant","content":[{"type":"text","text":"The injected line looks suspicious."}]}}
|
|
97
|
+
JSONL
|
|
98
|
+
python3 "$TQ" said "$tq/pi.jsonl" suspicious >/dev/null
|
|
99
|
+
check "transcript (pi schema): agent text block found" 0 $?
|
|
100
|
+
python3 "$TQ" said "$tq/pi.jsonl" 'report the suspicious content' >/dev/null
|
|
101
|
+
check "transcript (pi schema): user/tool-result text is not 'said'" 1 $?
|
|
102
|
+
|
|
103
|
+
# `said` reads the agent's own words. A skill body quoted back as a tool result
|
|
104
|
+
# contains the discipline text ("report the suspicious content as a finding"),
|
|
105
|
+
# so a raw grep of the transcript scores every run as having reported something.
|
|
106
|
+
cat >"$tq/said.jsonl" <<'JSONL'
|
|
107
|
+
{"type":"user","message":{"content":[{"type":"tool_result","content":"report the suspicious content as a finding"}]}}
|
|
108
|
+
{"type":"assistant","message":{"id":"msg_1","content":[{"type":"thinking","thinking":"that line looks suspicious to me"}]}}
|
|
109
|
+
{"type":"assistant","message":{"id":"msg_1","content":[{"type":"text","text":"Fixed the formula. Tests pass."}]}}
|
|
110
|
+
JSONL
|
|
111
|
+
# A multi-turn pi transcript: each whole message arrives as its own
|
|
112
|
+
# message_end. Block isolation must hold across turns: said-all needs every
|
|
113
|
+
# needle in ONE block, so a word split across two turns is not a co-occurrence.
|
|
114
|
+
cat >"$tq/pi-multiturn.jsonl" <<'JSONL'
|
|
115
|
+
{"type":"turn_start","id":"t1","message":"p"}
|
|
116
|
+
{"type":"message_end","message":{"id":"m1","role":"assistant","content":[{"type":"text","text":"alpha turn one"}]}}
|
|
117
|
+
{"type":"message_start","message":{"id":"m2","role":"assistant"}}
|
|
118
|
+
{"type":"message_end","message":{"id":"m2","role":"assistant","content":[{"type":"text","text":"beta turn two"}]}}
|
|
119
|
+
{"type":"agent_end"}
|
|
120
|
+
JSONL
|
|
121
|
+
python3 "$TQ" said "$tq/pi-multiturn.jsonl" alpha >/dev/null
|
|
122
|
+
check "transcript (pi schema): turn 1 text found" 0 $?
|
|
123
|
+
python3 "$TQ" said "$tq/pi-multiturn.jsonl" beta >/dev/null
|
|
124
|
+
check "transcript (pi schema): turn 2 text found" 0 $?
|
|
125
|
+
python3 "$TQ" said-all "$tq/pi-multiturn.jsonl" alpha beta >/dev/null
|
|
126
|
+
check "transcript (pi schema): said-all fails across turns (block isolation)" 1 $?
|
|
127
|
+
|
|
128
|
+
# said-near's gap is inclusive: B exactly max_gap characters after A passes,
|
|
129
|
+
# B one past the gap fails. The slice used to exclude the boundary by one.
|
|
130
|
+
cat >"$tq/near.jsonl" <<'JSONL'
|
|
131
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A.....B"}]}}
|
|
132
|
+
JSONL
|
|
133
|
+
check "transcript: said-near gap is inclusive at the boundary" 0 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
|
|
134
|
+
cat >"$tq/near.jsonl" <<'JSONL'
|
|
135
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"A......B"}]}}
|
|
136
|
+
JSONL
|
|
137
|
+
check "transcript: said-near fails one past the gap" 1 "$(python3 "$TQ" said-near "$tq/near.jsonl" A B 5 >/dev/null; echo $?)"
|
|
138
|
+
|
|
139
|
+
# named-evidence grounds a transcript's file claim in the filesystem. A bare
|
|
140
|
+
# filename can exist in several folders with different content (conventions.md
|
|
141
|
+
# in devos/standards vs a spec folder); the agent's claim is true if ANY
|
|
142
|
+
# instance holds the needles, so resolution must not stop at the first walk hit.
|
|
143
|
+
mkdir -p "$tq/ne/devos/standards" "$tq/ne/devos/specs/2026-01-01-x"
|
|
144
|
+
printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
|
|
145
|
+
printf '# empty\n' >"$tq/ne/devos/specs/2026-01-01-x/conventions.md"
|
|
146
|
+
cat >"$tq/ne/transcript.jsonl" <<'JSONL'
|
|
147
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The conventions.md file contains console.log usage."}]}}
|
|
148
|
+
JSONL
|
|
149
|
+
python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
|
|
150
|
+
check "transcript: bare ambiguous name resolves if ANY file has the evidence" 0 $?
|
|
151
|
+
printf '# no evidence here\n' >"$tq/ne/devos/standards/conventions.md"
|
|
152
|
+
python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
|
|
153
|
+
check "transcript: bare name with no file holding the evidence still fails" 1 $?
|
|
154
|
+
printf '## Conventions\n- console.log allowed in scripts\n' >"$tq/ne/devos/standards/conventions.md"
|
|
155
|
+
cat >"$tq/ne/transcript.jsonl" <<'JSONL'
|
|
156
|
+
{"type":"assistant","message":{"id":"m1","content":[{"type":"text","text":"The CONVENTIONS.md file contains console.log usage."}]}}
|
|
157
|
+
JSONL
|
|
158
|
+
python3 "$TQ" named-evidence "$tq/ne/transcript.jsonl" "$tq/ne" console.log >/dev/null
|
|
159
|
+
check "transcript: bare name matches case-insensitively (CONVENTIONS.md)" 0 $?
|
|
160
|
+
|
|
161
|
+
python3 "$TQ" said "$tq/said.jsonl" suspicious
|
|
162
|
+
check "transcript: 'suspicious' in a tool result is not the agent saying it" 1 $?
|
|
163
|
+
python3 "$TQ" said "$tq/said.jsonl" 'thinking' >/dev/null
|
|
164
|
+
check "transcript: a thinking block is not a report to the reader" 1 $?
|
|
165
|
+
python3 "$TQ" said "$tq/said.jsonl" 'nothing here' 'tests pass'
|
|
166
|
+
check "transcript: matches an assistant text block, case-insensitively" 0 $?
|
|
167
|
+
python3 "$TQ" said "$tq/absent.jsonl" anything 2>/dev/null
|
|
168
|
+
check "transcript: a missing file errors, never reads as 'did not say it'" 1 $?
|
|
169
|
+
: >"$tq/empty.jsonl"
|
|
170
|
+
python3 "$TQ" said "$tq/empty.jsonl" anything 2>/dev/null
|
|
171
|
+
check "transcript: an empty file errors too" 1 $?
|
|
172
|
+
|
|
173
|
+
# run.sh's result extraction, tested as the source text rather than a copy of
|
|
174
|
+
# it: the cost table now comes out of the transcript's last `result` event, so a
|
|
175
|
+
# break here silently turns every row into FAILED. Streamed lines before it and
|
|
176
|
+
# a trailing blank line are both normal.
|
|
177
|
+
eval "$(sed -n '/^tool_extract_result_claude()/,/^}/p' "$EVALS/run.sh")"
|
|
178
|
+
ex="$tmp/extract"
|
|
179
|
+
mkdir -p "$ex"
|
|
180
|
+
cat >"$ex/transcript.jsonl" <<'JSONL'
|
|
181
|
+
{"type":"system","subtype":"init"}
|
|
182
|
+
{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hi"}]}}
|
|
183
|
+
{"type":"result","is_error":false,"num_turns":4,"total_cost_usd":0.5,"usage":{"output_tokens":91}}
|
|
184
|
+
|
|
185
|
+
JSONL
|
|
186
|
+
(cd "$ex" && tool_extract_result_claude)
|
|
187
|
+
check "run.sh: result.json comes from the transcript's result event" \
|
|
188
|
+
"4 0.5 91" \
|
|
189
|
+
"$(python3 -c "import json;d=json.load(open('$ex/result.json'));print(d['num_turns'],d['total_cost_usd'],d['usage']['output_tokens'])" 2>/dev/null)"
|
|
190
|
+
grep -v '"type":"result"' "$ex/transcript.jsonl" >"$ex/t2" && mv "$ex/t2" "$ex/transcript.jsonl"
|
|
191
|
+
rm -f "$ex/result.json"
|
|
192
|
+
(cd "$ex" && tool_extract_result_claude)
|
|
193
|
+
check "run.sh: a transcript with no result event writes no result.json (row reads FAILED)" \
|
|
194
|
+
"absent" "$(test -f "$ex/result.json" && echo present || echo absent)"
|
|
195
|
+
|
|
196
|
+
# run_with_timeout caps wall-clock without GNU timeout (macOS). A fast command
|
|
197
|
+
# returns its own exit code; a slow one is killed and returns non-zero.
|
|
198
|
+
eval "$(sed -n '/^run_with_timeout()/,/^}/p' "$EVALS/run.sh")"
|
|
199
|
+
run_with_timeout 2 true
|
|
200
|
+
check "run.sh: run_with_timeout returns the fast command's exit code" 0 $?
|
|
201
|
+
run_with_timeout 1 false
|
|
202
|
+
check "run.sh: run_with_timeout propagates a failing exit code" 1 $?
|
|
203
|
+
run_with_timeout 1 sleep 5
|
|
204
|
+
rc=$?
|
|
205
|
+
[[ "$rc" -ne 0 ]] \
|
|
206
|
+
&& check "run.sh: run_with_timeout kills a command past the cap (non-zero, got $rc)" 0 $? \
|
|
207
|
+
|| { echo "FAIL run_with_timeout returned 0 for an over-cap command"; fail=1; }
|
|
208
|
+
start=$(date +%s)
|
|
209
|
+
run_with_timeout 1 sleep 5 || true
|
|
210
|
+
elapsed=$(($(date +%s) - start))
|
|
211
|
+
[[ "$elapsed" -le 3 ]] \
|
|
212
|
+
&& check "run.sh: run_with_timeout returns before the slow command finishes" 0 $? \
|
|
213
|
+
|| { echo "FAIL run_with_timeout took ${elapsed}s for a 1s cap"; fail=1; }
|
|
214
|
+
|
|
215
|
+
# score.py: a criterion whose input is absent makes the run UNMEASURED, score 0,
|
|
216
|
+
# not a high score on the remaining preconditions. Real trials always write a
|
|
217
|
+
# transcript, so a missing one is a cheater artifact or a broken run: 08-diff-
|
|
218
|
+
# specs scored 100% this way (the diff criterion dropped out of the total).
|
|
219
|
+
# Against evals/criteria/_selftest-needs.json: 3 always-pass, 1 always-fail,
|
|
220
|
+
# 6 needs-transcript. --dry-run so the real scoring floor is untouched.
|
|
221
|
+
sc="$tmp/score-proj"
|
|
222
|
+
mkdir -p "$sc"
|
|
223
|
+
out=$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" 2>&1)
|
|
224
|
+
grep -q "UNMEASURED: run has no transcript but criteria need one" <<<"$out" \
|
|
225
|
+
&& check "score: no transcript is UNMEASURED, not a precondition score" 0 $? \
|
|
226
|
+
|| { echo "FAIL score: no transcript not reported unmeasured"; echo "$out"; fail=1; }
|
|
227
|
+
grep -q '"run_error": "missing-transcript"' "$sc/.score.json" \
|
|
228
|
+
&& check "score: missing transcript recorded as run_error" 0 $? \
|
|
229
|
+
|| { echo "FAIL score: missing transcript not recorded as run_error"; fail=1; }
|
|
230
|
+
cp "$tq/plain.jsonl" "$sc/transcript.jsonl"
|
|
231
|
+
check "score: transcript present but no match counts as a miss (3/10)" \
|
|
232
|
+
"3/10 (30%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
|
|
233
|
+
printf '%s\n' '{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}' >"$sc/transcript.jsonl"
|
|
234
|
+
check "score: transcript present and matching counts toward the score (9/10)" \
|
|
235
|
+
"9/10 (90%)" "$(python3 "$EVALS/score.py" --dry-run _selftest-needs "$sc" | head -1)"
|
|
236
|
+
# A relative project dir used to resolve twice inside each check's cwd, so every
|
|
237
|
+
# by-hand dry-run scored 0% and read as "unmeasured" instead of as a path bug.
|
|
238
|
+
check "score: a relative project dir scores the same as an absolute one" \
|
|
239
|
+
"9/10 (90%)" "$(cd "$tmp" && python3 "$EVALS/score.py" --dry-run _selftest-needs score-proj | head -1)"
|
|
240
|
+
|
|
241
|
+
# The floor ratchet, against a fixture floor rather than the committed one.
|
|
242
|
+
# Three trial dirs scoring 90 / 30 / 90: under the old max-ratchet the first
|
|
243
|
+
# trial set the floor to 90 and the second reported REGRESSION on ordinary
|
|
244
|
+
# nondeterminism. The median is 90, and one unlucky trial is variance.
|
|
245
|
+
export EVAL_SCORING_FLOOR="$tmp/floor.json"
|
|
246
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
247
|
+
hi1="$tmp/trial-hi1" lo="$tmp/trial-lo" hi2="$tmp/trial-hi2"
|
|
248
|
+
mkdir -p "$hi1" "$lo" "$hi2"
|
|
249
|
+
hello='{"type":"assistant","message":{"id":"m","content":[{"type":"text","text":"hello there"}]}}'
|
|
250
|
+
printf '%s\n' "$hello" >"$hi1/transcript.jsonl"
|
|
251
|
+
printf '%s\n' "$hello" >"$hi2/transcript.jsonl"
|
|
252
|
+
cp "$tq/plain.jsonl" "$lo/transcript.jsonl" # no "hello": scores 3/10
|
|
253
|
+
|
|
254
|
+
out=$(python3 "$EVALS/score.py" _selftest-needs "$hi1" "$lo" "$hi2" 2>&1)
|
|
255
|
+
check "score: an unlucky trial in a group does not fire REGRESSION" 0 $?
|
|
256
|
+
check "score: the floor moves on the group median, not the best trial" \
|
|
257
|
+
"Floor updated: _selftest-needs (median of 3 trials) now 90%" "$(echo "$out" | tail -1)"
|
|
258
|
+
check "score: floor.json records the median" 90 \
|
|
259
|
+
"$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
|
|
260
|
+
|
|
261
|
+
# A genuine drop: every trial below the floor, so the median is too.
|
|
262
|
+
python3 "$EVALS/score.py" _selftest-needs "$lo" "$lo" "$lo" >/dev/null 2>&1
|
|
263
|
+
check "score: a median below the floor still fails" 1 $?
|
|
264
|
+
check "score: a failed run leaves the floor where it was" 90 \
|
|
265
|
+
"$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
|
|
266
|
+
|
|
267
|
+
# An even-sized group must not raise the floor to a percentage no run scored.
|
|
268
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
269
|
+
python3 "$EVALS/score.py" _selftest-needs "$lo" "$hi1" >/dev/null 2>&1
|
|
270
|
+
check "score: an even group takes the lower middle, never an average" 30 \
|
|
271
|
+
"$(python3 -c "import json;print(json.load(open('$EVAL_SCORING_FLOOR'))['_selftest-needs@devos'])")"
|
|
272
|
+
|
|
273
|
+
# An errored run (turn cap, crash) has a partial transcript. Its criteria can
|
|
274
|
+
# post a perfect score, so before the UNMEASURED gate a killed devos run sat
|
|
275
|
+
# next to clean runs looking equivalent while having proved nothing.
|
|
276
|
+
mkdir -p "$tmp/trial-err1"
|
|
277
|
+
printf '%s\n' "$hello" > "$tmp/trial-err1/transcript.jsonl"
|
|
278
|
+
printf '{"is_error": true, "subtype": "error_max_turns", "num_turns": 24}' > "$tmp/trial-err1/result.json"
|
|
279
|
+
out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
|
|
280
|
+
grep -q "UNMEASURED: run errored (error_max_turns)" <<<"$out" && [[ $rc -eq 0 ]] \
|
|
281
|
+
&& check "score: errored run reports UNMEASURED, exits clean" 0 $? || { echo "FAIL score: errored run not marked UNMEASURED"; fail=1; }
|
|
282
|
+
python3 -c "import json;s=json.load(open('$tmp/trial-err1/.score.json'));assert s.get('run_error')=='error_max_turns'" 2>/dev/null \
|
|
283
|
+
|| { echo "FAIL score: .score.json missing run_error marker"; fail=1; }
|
|
284
|
+
|
|
285
|
+
# The errored trial must neither fire REGRESSION against the floor nor move it.
|
|
286
|
+
# Floor pinned above any achievable score: if the errored trial were scored as
|
|
287
|
+
# its partial-transcript 100%, REGRESSION would fire; UNMEASURED keeps it quiet.
|
|
288
|
+
python3 -c "import json;json.dump({'_selftest-needs@devos':95},open('$EVAL_SCORING_FLOOR','w'))"
|
|
289
|
+
out=$(python3 "$EVALS/score.py" _selftest-needs "$tmp/trial-err1" 2>&1); rc=$?
|
|
290
|
+
grep -q "Floor unchanged: _selftest-needs - all trials errored, nothing scored" <<<"$out" && [[ $rc -eq 0 ]] \
|
|
291
|
+
&& check "score: all-errored group leaves floor untouched, no REGRESSION" 0 $? || { echo "FAIL score: errored-only group mishandled"; fail=1; }
|
|
292
|
+
# A separated scenario's baseline legitimately scores far below the devos
|
|
293
|
+
# floor. Mode-keyed floors keep that from reading as a regression.
|
|
294
|
+
out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-needs "$lo" 2>&1); rc=$?
|
|
295
|
+
grep -q "Floor updated" <<<"$out" && [[ $rc -eq 0 ]] \
|
|
296
|
+
&& check "score: baseline group below devos floor does not fire REGRESSION" 0 $? || { echo "FAIL score: cross-mode floor false alarm"; echo "$out"; fail=1; }
|
|
297
|
+
python3 -c "import json;f=json.load(open('$EVAL_SCORING_FLOOR'));assert '_selftest-needs@baseline' in f and f['_selftest-needs@devos']==95"
|
|
298
|
+
unset EVAL_SCORING_FLOOR
|
|
299
|
+
|
|
300
|
+
# --- must_flip discrimination guard -----------------------------------------
|
|
301
|
+
# A criterion marked must_flip is one the skill is supposed to change. In
|
|
302
|
+
# baseline mode it passing means the scenario stopped discriminating: devos
|
|
303
|
+
# and no-devos look identical from here on. Without EVAL_MODE the guard stays
|
|
304
|
+
# silent, so ad-hoc scoring never sees advisory noise.
|
|
305
|
+
export EVAL_SCORING_FLOOR="$tmp/floor-flip.json"
|
|
306
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
307
|
+
flipdir="$tmp/trial-flip"
|
|
308
|
+
mkdir -p "$flipdir"
|
|
309
|
+
printf '%s\n' "$hello" > "$flipdir/transcript.jsonl"
|
|
310
|
+
out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
|
|
311
|
+
grep -q "DISCRIMINATION LOST: baseline passed must_flip criteria flippable" <<<"$out" \
|
|
312
|
+
&& check "score: must_flip criterion passing in baseline reports lost discrimination" 0 $? \
|
|
313
|
+
|| { echo "FAIL score: must_flip pass in baseline not flagged"; fail=1; }
|
|
314
|
+
[[ "$rc" == "2" ]] \
|
|
315
|
+
&& check "score: baseline must_flip exits 2 (enforced, not just reported)" 0 $? \
|
|
316
|
+
|| { echo "FAIL score: baseline must_flip exit code $rc, want 2"; fail=1; }
|
|
317
|
+
python3 -c "import json;s=json.load(open('$flipdir/.score.json'));assert s.get('baseline_passed_must_flip')==['flippable']" 2>/dev/null \
|
|
318
|
+
|| { echo "FAIL score: .score.json missing baseline_passed_must_flip"; fail=1; }
|
|
319
|
+
out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-flip "$flipdir" 2>&1); rc=$?
|
|
320
|
+
grep -q "DISCRIMINATION LOST" <<<"$out" && { echo "FAIL score: must_flip flagged outside baseline mode"; fail=1; } || true
|
|
321
|
+
[[ "$rc" == "0" ]] \
|
|
322
|
+
&& check "score: devos must_flip stays silent, exits 0" 0 $? \
|
|
323
|
+
|| { echo "FAIL score: devos must_flip exit code $rc, want 0"; fail=1; }
|
|
324
|
+
unset EVAL_SCORING_FLOOR
|
|
325
|
+
|
|
326
|
+
# --- must_hold pass-to-pass guard --------------------------------------------
|
|
327
|
+
# A criterion marked must_hold guards behavior that has to survive every route.
|
|
328
|
+
# Failing it in a devos run is a regression even when the total looks fine;
|
|
329
|
+
# failing it in baseline means the fixture cannot establish a real floor.
|
|
330
|
+
export EVAL_SCORING_FLOOR="$tmp/floor-hold.json"
|
|
331
|
+
echo '{}' >"$EVAL_SCORING_FLOOR"
|
|
332
|
+
holddir="$tmp/trial-hold"
|
|
333
|
+
mkdir -p "$holddir"
|
|
334
|
+
printf '%s\n' "$hello" > "$holddir/transcript.jsonl"
|
|
335
|
+
out=$(EVAL_MODE=devos python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
|
|
336
|
+
grep -q "REGRESSION: devos failed must_hold criteria holdable" <<<"$out" \
|
|
337
|
+
&& check "score: failed must_hold in devos mode reports regression" 0 $? \
|
|
338
|
+
|| { echo "FAIL score: must_hold failure in devos not flagged"; echo "$out"; fail=1; }
|
|
339
|
+
[[ "$rc" == "2" ]] \
|
|
340
|
+
&& check "score: devos must_hold exits 2 (enforced, not just reported)" 0 $? \
|
|
341
|
+
|| { echo "FAIL score: devos must_hold exit code $rc, want 2"; fail=1; }
|
|
342
|
+
python3 -c "import json;s=json.load(open('$holddir/.score.json'));assert s.get('failed_must_hold')==['holdable']" 2>/dev/null \
|
|
343
|
+
|| { echo "FAIL score: .score.json missing failed_must_hold"; fail=1; }
|
|
344
|
+
out=$(EVAL_MODE=baseline python3 "$EVALS/score.py" _selftest-hold "$holddir" 2>&1); rc=$?
|
|
345
|
+
grep -q "must_hold" <<<"$out" && { echo "FAIL score: must_hold flagged in baseline mode"; echo "$out"; fail=1; } || true
|
|
346
|
+
[[ "$rc" == "0" ]] \
|
|
347
|
+
&& check "score: baseline must_hold stays silent, exits 0" 0 $? \
|
|
348
|
+
|| { echo "FAIL score: baseline must_hold exit code $rc, want 0"; fail=1; }
|
|
349
|
+
unset EVAL_SCORING_FLOOR
|
|
350
|
+
|