ll-skills 2.0.2 → 3.0.0

package/scripts/evals/cases/executor-block/assert.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# The executor returns exactly one block with the seven fields, and the TDD order holds.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$OUT_TXT" '^### M1'          'the return opens with the ### M1 heading'
+for f in built commits commands deviations questions backlog not_verified; do
+  contains "$OUT_TXT" "^${f}:" "the return carries the field ${f}:"
+done
+
+# TDD order over the whole history: a test(M1) commit exists and precedes the first feat(M1).
+log="$(git -C "$WORK" log --format=%s --reverse)"
+t="$(printf '%s\n' "$log" | grep -n '^test(M1): ' | head -1 | cut -d: -f1)"
+f="$(printf '%s\n' "$log" | grep -n '^feat(M1): ' | head -1 | cut -d: -f1)"
+if [ -n "$t" ] && [ -n "$f" ] && [ "$t" -lt "$f" ]; then
+  ok "git log shows test(M1) at #$t before feat(M1) at #$f"
+else
+  fail "git log does not show a test(M1) commit before a feat(M1) commit (test=${t:-none} feat=${f:-none})"
+fi
+
+# The executor owns no state file.
+base="$(base_sha "$WORK")"
+if [ -n "$base" ]; then
+  touched="$(git -C "$WORK" diff --name-only "$base" HEAD -- PROGRESS.md PLAN.md ROADMAP.md BACKLOG.md decisions)"
+  if [ -z "$touched" ]; then
+    ok 'no state file was committed by the executor'
+  else
+    fail "the executor committed a state file: $(printf '%s' "$touched" | tr '\n' ' ')"
+  fi
+fi
+
+finish

package/scripts/evals/cases/executor-block/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 40,
+  "history": true,
+  "min_pass": 2,
+  "agent": "ll-executor",
+  "permission_mode": "bypassPermissions",
+  "note": "The fixture history already builds M1 (test(M1) then feat(M1)), so the red test passes on its first run and the contract says the executor returns BLOCKED. The case scores the return block, not the code."
+}

package/scripts/evals/cases/executor-block/prompt.txt

@@ -0,0 +1,14 @@
+MILESTONE     M1 — end-to-end tracer
+PLAN          {{WORK}}/phases/07/PLAN.md            (read whole before acting; your entry is the contract)
+CONTEXT       none — no CODE-CONTEXT.md for this phase; use read_first in the plan
+FILES         {{WORK}}/src/a.ts, {{WORK}}/test/a.test.ts
+WAVE          1 of 3 · alone in this wave · previous blocks: {{WORK}}/PROGRESS.md "## Phase 07"
+TDD           yes — behavior cases in PLAN, milestone M1 · test first, commit test(M1) before feat(M1)
+ACCEPTANCE    npm test -- a.test.ts — run from {{WORK}} after the last commit
+MODEL         contract milestone → opus/high
+DEC RESERVED  DEC-0042, DEC-0043  (cite only these ids in questions:; never create a decision)
+INPUTS        {{WORK}}/src/a.ts, {{WORK}}/test/a.test.ts  (checked: exist)
+DO NOT        write PROGRESS/PLAN/ROADMAP/BACKLOG/decisions; commit outside FILES; push; weaken a test;
+              spawn an agent; cd; relative paths; grep a directory that holds a .env
+RETURN        the `### M1` block in your fixed format (built, commits, commands, deviations, questions,
+              backlog, not_verified), at most 1,500 tokens. Nothing before or after it.

package/scripts/evals/cases/goal-autonomous/assert.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# `/ll-goal --autonomous "<objective>"`: one pasted /goal text for the whole delivery,
+# docs/GOAL.md written with `mode: autonomous` and `phase: all`, nothing asked, no skill started.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$OUT_TXT" '/goal'                   'the answer carries the /goal text to paste'
+# EXECUTION carries `ll-auto` with `--auto-decision`; the objective and `--verify all` may sit
+# between them and the text wraps, so the answer is read flattened.
+if tr '\n' ' ' < "$OUT_TXT" 2>/dev/null | grep -Eq 'll-auto.{0,80}--auto-decision'; then
+  ok 'EXECUTION runs the delivery with ll-auto --auto-decision'
+else
+  fail 'EXECUTION runs the delivery with ll-auto --auto-decision'
+fi
+
+# The pasted block is what the owner copies: from the /goal line to the `▶ Next` line, or to the
+# end when there is none. The goal text is a pointer, never a copy of PLAN.md: it stays small.
+block="$(awk '/\/goal/ { on = 1 } on && /▶ Next/ { exit } on { print }' "$OUT_TXT" 2>/dev/null)"
+n="$(printf '%s' "$block" | wc -c)"
+if [ "$n" -ge 400 ] && [ "$n" -le 4000 ]; then
+  ok "the pasted /goal text measures $n chars (400..4000)"
+else
+  fail "the pasted /goal text measures $n chars, outside 400..4000"
+fi
+
+if [ -f "$WORK/docs/GOAL.md" ]; then ok 'docs/GOAL.md exists in the work tree'
+else fail 'docs/GOAL.md exists in the work tree'; fi
+contains "$WORK/docs/GOAL.md" '^mode: autonomous$' 'docs/GOAL.md frontmatter carries mode: autonomous'
+contains "$WORK/docs/GOAL.md" '^phase: all$'       'docs/GOAL.md frontmatter carries phase: all (the whole delivery)'
+
+no_tool_use "$OUT_JSON" Skill           'no Skill tool call anywhere in the capture'
+no_tool_use "$OUT_JSON" AskUserQuestion 'no AskUserQuestion tool call anywhere in the capture'
+
+finish

package/scripts/evals/cases/goal-autonomous/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 20,
+  "history": false,
+  "min_pass": 1,
+  "agent": null,
+  "permission_mode": "bypassPermissions",
+  "note": "Own fixture, not the shared scripts/fixtures/project: a healthy repo with PLAN.md, ROADMAP.md (07 and 08 PLANNED), a consistent PROGRESS.md board, no phases/NN/PLAN.md yet and `npm test` exit 0. The shared fixture is broken on purpose (plan-lint verdict fail, milestones marked passes: true with no commit, no test runner) and ll-goal step 1 is right to stop there — `a plan that fails there does not become a goal` — so it can never prove the autonomous branch. Proves: one pasted /goal text for the whole delivery with EXECUTION on ll-auto --auto-decision, docs/GOAL.md with `mode: autonomous` and `phase: all` committed, nothing asked and no skill started. max_turns 20 (B-018 rule: above the highest count real reps show — 17 here): the run reads PLAN/ROADMAP/PROGRESS, calls the ll-auto detect helper, writes and commits docs/GOAL.md, and the reported num_turns counts the skill's `!` preprocessor Bash calls together with the model's own tool calls."
+}

package/scripts/evals/cases/goal-autonomous/fixture/PLAN.md

@@ -0,0 +1,42 @@
+# PLAN — reconciliation service
+
+## §0 Precedence
+
+This file is self-contained and the ONLY entry of the work. §2 > §3 > §6 > §7. Decisions in §3 are a
+contract — do not re-litigate. Project CLAUDE.md > this file > phase plans > briefs.
+
+## §1 Objective and truths
+
+Every provider event is reconciled exactly once and every mismatch is visible to the operator.
+
+- T1 A provider batch produces one row per event. — `npm test` exit 0
+- T2 A replayed batch adds no rows. — `npm test` exit 0
+
+Requirements: REQ-k (phase 07), REQ-m (phase 08).
+
+## §2 Invariants
+
+I-01 Never delete, disable or weaken a test or an acceptance criterion. [owner, 2026-08-20]
+I-02 Do not fill a gap with a plausible interpretation: report and ask. [owner, CLAUDE.md]
+
+## §3 Owner decisions
+
+| id | question | decision | by | date | against recommendation? | reversible? |
+|---|---|---|---|---|---|---|
+| DEC-0041 | mismatch unit | cents, integer | owner | 2026-08-22 | no | 1 commit |
+
+## §6 Global acceptance
+
+CA-01 — WHEN a batch is ingested THE SYSTEM SHALL write one row per event · `npm test` exit 0
+CA-02 — WHEN a mismatch is found THE SYSTEM SHALL report it · `npm test` exit 0
+
+## §7 Execution protocol
+
+Models per role: session fable/high · scout sonnet/medium · contract executor opus/high · mechanical
+executor sonnet/medium · verifier opus/high · reviewer opus/medium. Max 3 executors per wave on
+disjoint files. Commit per path after each milestone; push only with everything green (push is not a
+deploy here). TDD on by default; exceptions: config and glue.
+
+## §8 Phases
+
+Phases live in ROADMAP.md. Current phase: 07.

package/scripts/evals/cases/goal-autonomous/fixture/PROGRESS.md

@@ -0,0 +1,20 @@
+# PROGRESS — reconciliation service
+
+<!-- ll-state -->
+phase: 07
+milestones:
+<!-- /ll-state -->
+
+## Rules for all agents
+
+- Only the main session writes this file. Executors return blocks; the session appends them.
+- Never delete, disable or weaken a test or an acceptance criterion.
+- Commit per milestone with `type(Mn): what`.
+
+## Epilogue — phase 06 — 2026-08-30
+
+passed: M1, M2 (2/2) — provider adapters shipped, `npm test` green.
+left: none. waiting: none.
+verification: APPROVED.
+
+## Phase 07

package/scripts/evals/cases/goal-autonomous/fixture/ROADMAP.md

@@ -0,0 +1,29 @@
+# ROADMAP — reconciliation service
+
+| phase | name | depends_on | requirements | state |
+|---|---|---|---|---|
+| 05 | payment intake | — | REQ-a | DONE (docs/history/v1.0) |
+| 06 | provider adapters | 05 | REQ-b | DONE (docs/history/v1.0) |
+| 07 | billing reconciliation | 05, 06 | REQ-k | PLANNED |
+| 08 | reconciliation reporting | 07 | REQ-m | PLANNED |
+
+## Phase 07 — billing reconciliation
+
+Objective: every provider event lands in `billing_events` exactly once and mismatches are visible.
+
+Success criteria:
+- SC-01 The reconciliation job ingests a provider batch and writes one row per event.
+- SC-02 A replayed batch produces no duplicate rows.
+- SC-03 Cent-level mismatches are reported instead of silently dropped.
+
+Deferred ideas: customer portal; accounting export.
+
+## Phase 08 — reconciliation reporting
+
+Objective: the operator reads yesterday's reconciliation without opening the database.
+
+Success criteria:
+- SC-01 A daily report lists ingested, tolerated and rejected rows.
+- SC-02 The report is reachable from the operator console.
+
+Deferred ideas: CSV export.

package/scripts/evals/cases/goal-autonomous/fixture/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "reconciliation-service",
+  "version": "0.6.0",
+  "private": true,
+  "scripts": {
+    "test": "node --test test/*.test.js"
+  }
+}

package/scripts/evals/cases/goal-autonomous/fixture/src/money.js

@@ -0,0 +1,6 @@
+// Cent-level arithmetic: DEC-0041 — money is an integer number of cents.
+function toCents(amount) {
+  return Math.round(Number(amount) * 100);
+}
+
+module.exports = { toCents };

package/scripts/evals/cases/goal-autonomous/fixture/test/reconcile.test.js

@@ -0,0 +1,8 @@
+const test = require('node:test');
+const assert = require('node:assert');
+
+const { toCents } = require('../src/money.js');
+
+test('amounts are compared in integer cents', () => {
+  assert.strictEqual(toCents('10.05'), 1005);
+});

package/scripts/evals/cases/goal-autonomous/prompt.txt

@@ -0,0 +1 @@
+/ll-goal --autonomous "Deliver phases 07 and 08"

package/scripts/evals/cases/implement-review-gate/assert.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# The review gate opens wave 1: PLAN-REVIEW.md exists, and no feat( commit of this run
+# predates it. Only phase 07 is planned.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+REVIEW="phases/07/PLAN-REVIEW.md"
+
+if [ -f "$WORK/$REVIEW" ]; then
+  ok "$REVIEW exists"
+else
+  fail "$REVIEW does not exist: the gate before wave 1 was not written"
+fi
+
+base="$(base_sha "$WORK")"
+# Only commits this run added count; the fixture history already carries feat( commits.
+range="${base:+$base..HEAD}"
+first_feat="$(git -C "$WORK" log ${range:+"$range"} --reverse --format='%ct %s' | grep -m1 ' feat(' | cut -d' ' -f1)"
+
+if [ -z "$first_feat" ]; then
+  ok 'this run committed no feat(: nothing could precede the gate'
+else
+  # When the review was committed, git dates it; otherwise fall back to the file mtime.
+  review_ct="$(git -C "$WORK" log --diff-filter=A --format=%ct -- "$REVIEW" | tail -1)"
+  [ -n "$review_ct" ] || review_ct="$(stat -c %Y "$WORK/$REVIEW" 2>/dev/null)"
+  if [ -n "$review_ct" ] && [ "$review_ct" -le "$first_feat" ]; then
+    ok "the gate ($review_ct) precedes the first feat( of this run ($first_feat)"
+  else
+    fail "a feat( commit at $first_feat precedes the review gate at ${review_ct:-none}"
+  fi
+fi
+
+no_path "$WORK/phases/08" 'nothing was written under phases/08/'
+
+finish

package/scripts/evals/cases/implement-review-gate/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 45,
+  "history": true,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "bypassPermissions",
+  "note": "No executor is dispatched until phases/07/PLAN-REVIEW.md is on disk; and one invocation plans one phase, so phases/08/ must stay untouched."
+}

package/scripts/evals/cases/implement-review-gate/prompt.txt

@@ -0,0 +1 @@
+implementa a fase 7

package/scripts/evals/cases/implement-stops-at-next/assert.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# "▶ Next" ends the turn: no tool call follows it.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+verdict="$(node -e '
+  const fs = require("fs");
+  let data; try { data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); }
+  catch { process.stdout.write("unparseable out.json"); process.exit(0); }
+  const events = Array.isArray(data) ? data : [data];
+
+  let nextAt = -1;          // index of the assistant text block carrying the marker
+  let toolAfter = null;     // the first tool_use seen after it
+  events.forEach((ev, i) => {
+    if (!ev || ev.type !== "assistant") return;   // hook context and tool results are not assistant text
+    const content = ev.message && ev.message.content;
+    if (!Array.isArray(content)) return;
+    for (const b of content) {
+      if (!b) continue;
+      if (b.type === "text" && /▶ Next/.test(b.text || "")) { if (nextAt < 0) nextAt = i; }
+      else if (b.type === "tool_use" && nextAt >= 0 && !toolAfter) toolAfter = b.name;
+    }
+  });
+
+  if (nextAt < 0) { process.stdout.write("no assistant text carries the ▶ Next marker"); process.exit(0); }
+  if (toolAfter)  { process.stdout.write("a " + toolAfter + " tool call follows the ▶ Next line"); process.exit(0); }
+  process.stdout.write("");
+' "$OUT_JSON")"
+
+if [ -z "$verdict" ]; then
+  ok 'the ▶ Next line is the last thing the invocation does'
+else
+  fail "$verdict"
+fi
+
+contains "$OUT_TXT" '/clear' 'the next command is handed over with /clear'
+
+finish

package/scripts/evals/cases/implement-stops-at-next/case.json

@@ -0,0 +1,9 @@
+{
+  "max_turns": 30,
+  "history": true,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "bypassPermissions",
+  "reuse": "implement-review-gate",
+  "note": "Scores the same capture as implement-review-gate — no second claude call. Run both cases together, or this one reports the missing source."
+}

package/scripts/evals/cases/implement-stops-at-next/prompt.txt

@@ -0,0 +1 @@
+implementa a fase 7

package/scripts/evals/cases/preamble-no-ritual/assert.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# The typo is fixed and no ceremony file is born.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$WORK/src/a.ts" 'receive' 'src/a.ts now spells receive'
+absent   "$WORK/src/a.ts" 'recieve' 'no occurrence of the typo is left'
+
+no_path "$WORK/PROGRESS.md"     'no PROGRESS.md was created'
+no_path "$WORK/VERIFICATION.md" 'no VERIFICATION.md was created'
+no_path "$WORK/phases"          'no phases/ was created'
+no_path "$WORK/PLAN.md"         'no PLAN.md was created'
+
+no_tool_use "$OUT_JSON" Skill 'no Skill tool call anywhere in the capture'
+
+finish

package/scripts/evals/cases/preamble-no-ritual/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 6,
+  "history": false,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "bypassPermissions",
+  "note": "SMALL forbids the ritual: no spec, no plan, no PROGRESS, no VERIFICATION, no subagent for what fits in three calls, and no skill started on the session's behalf."
+}

package/scripts/evals/cases/preamble-no-ritual/fixture/README.md

@@ -0,0 +1,3 @@
+# events
+
+One module. One typo.

package/scripts/evals/cases/preamble-no-ritual/fixture/src/a.ts

@@ -0,0 +1,3 @@
+export function recieveEvent(id: string): string {
+  return `recieved:${id}`;
+}

package/scripts/evals/cases/preamble-no-ritual/prompt.txt

@@ -0,0 +1 @@
+corrige o typo 'recieve' em src/a.ts

package/scripts/evals/cases/router-execute/assert.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# "implementa a fase N" with a milestone at passes:false gets the /ll-implement command to paste.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+# The skill is named with the phase number; 7 and 07 both count.
+first_text_contains "$OUT_JSON" '/ll-implement 0?7' 'the first assistant message names /ll-implement 7 to paste'
+
+no_tool_use "$OUT_JSON" Skill 'no Skill tool call anywhere in the capture'
+
+finish

package/scripts/evals/cases/router-execute/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 1,
+  "history": false,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "acceptEdits",
+  "note": "max_turns 1 keeps the case cheap: the session names the /ll-implement 7 command to paste and starts no skill, read out of the first assistant message. Shared fixture: phases/07/PLAN.md plus a PROGRESS.md board with M2 passes:false."
+}

package/scripts/evals/cases/router-execute/prompt.txt

@@ -0,0 +1 @@
+implementa a fase 7

package/scripts/evals/cases/router-research/assert.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# "pesquise" gets the command to paste, not a skill started on the session's behalf.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+first_text_contains "$OUT_JSON" '/ll-research' 'the first assistant message names the /ll-research command to paste'
+
+no_tool_use "$OUT_JSON" Skill 'no Skill tool call anywhere in the capture'
+
+finish

package/scripts/evals/cases/router-research/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 1,
+  "history": false,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "acceptEdits",
+  "note": "max_turns 1 keeps the case cheap: the session names the /ll-research command to paste and starts no skill, read out of the first assistant message before any tool spending. Own fixture without PROGRESS.md so the answer measures one signal only: the word pesquise pointing at /ll-research."
+}

package/scripts/evals/cases/router-research/fixture/README.md

@@ -0,0 +1,3 @@
+# Ledger service
+
+A service that reconciles provider events against the ledger. No queue is wired yet.

package/scripts/evals/cases/router-research/prompt.txt

@@ -0,0 +1 @@
+pesquise as opções de fila para Node

package/scripts/evals/cases/router-small/assert.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# SMALL: verb + addressable target -> answer with a number, write nothing, start no skill.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$OUT_TXT" '[0-9]+' 'the answer carries a number'
+
+dirty="$(git -C "$WORK" status --porcelain)"
+if [ -z "$dirty" ]; then
+  ok 'git status --porcelain is empty: nothing was written'
+else
+  fail "the working tree was changed: $(printf '%s' "$dirty" | tr '\n' ' ')"
+fi
+
+no_path "$WORK/PROGRESS.md" 'no PROGRESS.md was created'
+no_path "$WORK/phases"      'no phases/ was created'
+
+no_tool_use "$OUT_JSON" Skill 'no Skill tool call anywhere in the capture'
+
+finish

package/scripts/evals/cases/router-small/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 6,
+  "history": false,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "acceptEdits",
+  "note": "Own fixture, deliberately without PROGRESS.md: the RESUME rule fires on the first turn in a repo that has one, and would compete with SMALL. The session answers directly, writes nothing and starts no skill."
+}

package/scripts/evals/cases/router-small/fixture/README.md

@@ -0,0 +1,17 @@
+# Ledger service
+
+A service that reconciles provider events against the ledger.
+
+## Install
+
+    npm install
+
+## Run
+
+    npm start
+
+## Test
+
+    npm test
+
+The reconciliation job runs every hour.

package/scripts/evals/cases/router-small/prompt.txt

@@ -0,0 +1 @@
+conta as linhas de README.md

package/scripts/evals/cases/scout-no-plan/assert.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# The scout writes CODE-CONTEXT.md within its cap and never opens the project PLAN.md.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+CTX="$WORK/phases/07/CODE-CONTEXT.md"
+
+if [ -f "$CTX" ]; then
+  n="$(wc -l < "$CTX")"
+  ok "phases/07/CODE-CONTEXT.md exists ($n lines)"
+  if [ "$n" -le 120 ]; then ok "the file is within its 120-line cap"; else fail "CODE-CONTEXT.md has $n lines, cap is 120"; fi
+else
+  fail 'phases/07/CODE-CONTEXT.md was not written'
+fi
+
+# No Read tool_use whose file_path is the project PLAN.md at the root of the work tree.
+offenders="$(node -e '
+  const fs = require("fs");
+  const work = process.argv[2];
+  let data; try { data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); } catch { process.exit(0); }
+  const events = Array.isArray(data) ? data : [data];
+  const hits = [];
+  for (const ev of events) {
+    const content = ev && ev.message && ev.message.content;
+    if (!Array.isArray(content)) continue;
+    for (const b of content) {
+      if (!b || b.type !== "tool_use" || b.name !== "Read") continue;
+      const p = (b.input && b.input.file_path) || "";
+      if (p === work + "/PLAN.md" || p === "PLAN.md" || p === "./PLAN.md") hits.push(p);
+    }
+  }
+  process.stdout.write(hits.join(" "));
+' "$OUT_JSON" "$WORK")"
+
+if [ -z "$offenders" ]; then
+  ok 'no Read of the project PLAN.md at the work-tree root'
+else
+  fail "the scout read the project PLAN.md: $offenders"
+fi
+
+finish

package/scripts/evals/cases/scout-no-plan/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 30,
+  "history": true,
+  "min_pass": 2,
+  "agent": "ll-scout",
+  "permission_mode": "bypassPermissions",
+  "note": "The scout must not read the project PLAN.md at the repository root. The phase plan is not scored here."
+}

package/scripts/evals/cases/scout-no-plan/prompt.txt

@@ -0,0 +1,8 @@
+PHASE         07 — every provider event lands in `billing_events` exactly once and mismatches are visible   (only this phase)
+MODEL         sonnet / medium — always, contract phases included
+FILES         create: {{WORK}}/src/ingest.ts · change: {{WORK}}/src/pay.ts
+SYMBOLS       none
+OUTPUT        {{WORK}}/phases/07/CODE-CONTEXT.md — at most 120 lines, Write not heredoc
+DO NOT        read the project PLAN.md, PROGRESS.md, phases/*/PLAN.md or decisions/; propose a plan;
+              run tests or builds
+RETURN        your fixed block, at most 10 lines: path, files classified, analogs, readers, assumed values

package/scripts/evals/cases/verifier-weakened-test/assert.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# A test whose assertion was removed without a DEC must not ride in an approval.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$OUT_TXT" 'REJECTED'       'the return carries the verdict REJECTED'
+contains "$OUT_TXT" 'test/a\.test\.ts' 'the return names test/a.test.ts'
+
+if [ -f "$WORK/phases/07/VERIFICATION.md" ]; then
+  ok 'phases/07/VERIFICATION.md was written'
+else
+  fail 'phases/07/VERIFICATION.md was not written at the path the brief gave'
+fi
+
+# The verifier fixes nothing: the weakened test is still weakened.
+absent "$WORK/test/a.test.ts" 'expect\(' 'the verifier did not repair the test it flagged'
+
+finish

package/scripts/evals/cases/verifier-weakened-test/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 40,
+  "history": true,
+  "min_pass": 2,
+  "agent": "ll-verifier",
+  "permission_mode": "bypassPermissions",
+  "note": "setup.sh commits a feat(M2) that strips the only expect() from test/a.test.ts, with no DEC id anywhere. The process check must catch it."
+}

package/scripts/evals/cases/verifier-weakened-test/prompt.txt

@@ -0,0 +1,13 @@
+MODE          phase
+ROADMAP       {{WORK}}/ROADMAP.md — section "Phase 07"
+PLAN          {{WORK}}/phases/07/PLAN.md — truths:, milestones, acceptance:, verification:, ## Errata
+DECISIONS     {{WORK}}/decisions/DEC-0041-cents.md
+CRITERIA      SC-01 The reconciliation job ingests a provider batch and writes one row per event.
+              SC-02 A replayed batch produces no duplicate rows.
+              SC-03 Cent-level mismatches are reported instead of silently dropped.
+              SC-04 The operator can list tolerated rows for a day.
+SLICE         branch main · the whole history · worktree {{WORK}}
+OUTPUT        {{WORK}}/phases/07/VERIFICATION.md — Write, not heredoc
+DO NOT        open PROGRESS.md before every state is written; edit code, tests, plan or passes;
+              run the whole suite to prove one criterion
+RETURN        your fixed block, at most 20 lines: verdict, product/process, state counts, BLOCKS, gaps

package/scripts/evals/cases/verifier-weakened-test/setup.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Commits a feat(M2) that weakens test/a.test.ts: the only expect() is dropped and the
+# test becomes unconditional. No DEC id in the message and none under decisions/.
+set -euo pipefail
+
+WORK="${1:?usage: setup.sh <workdir>}"
+
+cat > "$WORK/test/a.test.ts" <<'TS'
+import { reconcile } from '../src/a';
+
+test('reconcile dedupes a replayed batch', () => {
+  reconcile(['e1', 'e1', 'e2']);
+});
+TS
+
+git -C "$WORK" add test/a.test.ts
+GIT_AUTHOR_DATE="2026-09-10T10:00:00+00:00" GIT_COMMITTER_DATE="2026-09-10T10:00:00+00:00" \
+  git -C "$WORK" -c user.name=fixture -c user.email=fixture@example.com -c commit.gpgsign=false \
+  commit -q -m "feat(M2): simplify the reconcile assertion"