ll-skills 2.0.2 → 3.0.0

package/scripts/evals/fixtures/manual-contract/out.json

@@ -0,0 +1,29 @@
+[
+  {
+    "type": "system",
+    "subtype": "init",
+    "session_id": "manual-contract-fixture",
+    "model": "claude-sonnet-5"
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+        }
+      ]
+    }
+  },
+  {
+    "type": "result",
+    "subtype": "success",
+    "is_error": false,
+    "duration_ms": 1820,
+    "num_turns": 1,
+    "total_cost_usd": 0.0091,
+    "result": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+  }
+]

package/scripts/evals/fixtures/manual-contract/out.txt

@@ -0,0 +1,5 @@
+Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:
+
+/ll-research compare authentication providers for the signup flow
+
+▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`

package/scripts/evals/fixtures/manual-contract/with-skill.json

@@ -0,0 +1,46 @@
+[
+  {
+    "type": "system",
+    "subtype": "init",
+    "session_id": "manual-contract-fixture-with-skill",
+    "model": "claude-sonnet-5"
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+        }
+      ]
+    }
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "tool_use",
+          "id": "toolu_01manualcontract",
+          "name": "Skill",
+          "input": {
+            "skill": "ll-research",
+            "args": "compare authentication providers for the signup flow"
+          }
+        }
+      ]
+    }
+  },
+  {
+    "type": "result",
+    "subtype": "success",
+    "is_error": false,
+    "duration_ms": 2140,
+    "num_turns": 2,
+    "total_cost_usd": 0.0134,
+    "result": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+  }
+]

package/scripts/evals/lib/assert.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Sourced by every cases/<id>/assert.sh.
+#
+# Contract of an assert script:
+#   bash assert.sh <workdir> <out.json> <out.txt>   -> exit 0 pass, 1 fail
+# It prints one line per check: "ok: <what>" or "FAIL: <what>".
+# run.sh reads the first FAIL line into the results table.
+#
+# Provided helpers:
+#   ok <msg>                     record a passed check
+#   fail <msg>                   record a failed check
+#   check <cond-exit> <msg>      record from an exit code already computed
+#   contains <file> <regex> <msg>    grep -Eq
+#   absent <file> <regex> <msg>      grep -Eq must not match
+#   no_path <path> <msg>         path must not exist
+#   no_tool_use <out.json> <tool-name> <msg>   no tool_use block named <tool-name> anywhere
+#   finish                       exit with the verdict
+
+EVAL_FAILURES=0
+
+ok()   { printf 'ok: %s\n' "$1"; }
+fail() { printf 'FAIL: %s\n' "$1"; EVAL_FAILURES=$((EVAL_FAILURES + 1)); }
+
+check() { # check <exit-code> <msg>
+  if [ "$1" -eq 0 ]; then ok "$2"; else fail "$2"; fi
+}
+
+contains() { # contains <file> <extended-regex> <msg>
+  if [ -f "$1" ] && grep -Eq -- "$2" "$1"; then ok "$3"; else fail "$3"; fi
+}
+
+absent() { # absent <file> <extended-regex> <msg>
+  if [ ! -f "$1" ] || ! grep -Eq -- "$2" "$1"; then ok "$3"; else fail "$3"; fi
+}
+
+no_path() { # no_path <path> <msg>
+  if [ ! -e "$1" ]; then ok "$2"; else fail "$2"; fi
+}
+
+first_line() { # first_line <file> -> first non-empty line
+  [ -f "$1" ] || return 0
+  grep -m1 -v '^[[:space:]]*$' "$1" 2>/dev/null || true
+}
+
+# The regime is stated "before doing anything": it belongs to the first assistant
+# message of the run, not to the final answer. This reads it out of the capture.
+first_text() { # first_text <out.json> -> the whole first assistant message
+  node "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/extract.js" "$1" first_text 2>/dev/null || true
+}
+
+first_text_line() { # first_text_line <out.json> -> first non-empty line of the first assistant text
+  first_text "$1" | grep -m1 -v '^[[:space:]]*$' || true
+}
+
+first_text_contains() { # first_text_contains <out.json> <extended-regex> <msg>
+  if first_text "$1" | grep -Eq -- "$2"; then ok "$3"; else fail "$3"; fi
+}
+
+# no_tool_use <out.json> <tool-name> <msg>
+# Scans every assistant event's message.content for a tool_use block whose
+# `name` equals <tool-name>. Passes when none is found; fails naming the
+# first hit (the manual contract: no skill is started by a tool call).
+# A capture that is missing or is not readable JSON proves nothing: it fails
+# (B-003/B-020 — a rep whose out.json never landed used to score PASS).
+no_tool_use() {
+  local file="$1" name="$2" msg="$3" hit rc
+  if [ ! -f "$file" ]; then
+    fail "$msg: no capture at $file"
+    return
+  fi
+  hit="$(node -e '
+    const fs = require("fs");
+    let data;
+    try { data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); }
+    catch { process.exit(3); }
+    const events = Array.isArray(data) ? data : [data];
+    const wanted = process.argv[2];
+    for (const ev of events) {
+      if (!ev || ev.type !== "assistant") continue;
+      const content = ev.message && ev.message.content;
+      if (!Array.isArray(content)) continue;
+      for (const b of content) {
+        if (b && b.type === "tool_use" && b.name === wanted) {
+          process.stdout.write(b.name);
+          process.exit(0);
+        }
+      }
+    }
+  ' "$file" "$name" 2>/dev/null)"
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    fail "$msg: the capture at $file is not readable JSON"
+  elif [ -z "$hit" ]; then
+    ok "$msg"
+  else
+    fail "$msg: found a $hit tool_use call"
+  fi
+}
+
+base_sha() { # base_sha <workdir> -> the HEAD recorded before the run, or empty
+  [ -f "$1/.eval-base-sha" ] && cat "$1/.eval-base-sha"
+}
+
+finish() {
+  if [ "$EVAL_FAILURES" -eq 0 ]; then exit 0; fi
+  exit 1
+}

package/scripts/evals/lib/extract.js

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+'use strict';
+
+// Reads one field of the `type: "result"` element of a `claude --output-format json`
+// capture. The capture is either a single object or an array of events
+// (system/init … assistant … result); only the result element carries the totals.
+//
+//   node extract.js <out.json> [field]     field defaults to "result"
+//
+// Exits 1 when the file is not JSON or has no result element, so the caller can
+// tell "the run produced nothing" from "the run produced an empty answer".
+
+const fs = require('fs');
+
+const file = process.argv[2];
+const field = process.argv[3] || 'result';
+
+let raw;
+try {
+  raw = fs.readFileSync(file, 'utf8');
+} catch (e) {
+  process.stderr.write(`extract: cannot read ${file}: ${e.message}\n`);
+  process.exit(1);
+}
+
+let data;
+try {
+  data = JSON.parse(raw);
+} catch {
+  process.stderr.write(`extract: ${file} is not valid JSON\n`);
+  process.exit(1);
+}
+
+const events = Array.isArray(data) ? data : [data];
+
+const assistantTexts = () => {
+  const out = [];
+  for (const ev of events) {
+    if (!ev || ev.type !== 'assistant') continue;
+    const content = ev.message && ev.message.content;
+    if (!Array.isArray(content)) continue;
+    const t = content.filter((b) => b && b.type === 'text').map((b) => b.text).join('\n');
+    if (t.trim()) out.push(t);
+  }
+  return out;
+};
+
+// Pseudo-field: the first non-empty assistant message. The regime line is stated
+// "before doing anything", so it lives in the first turn, not in the final result.
+// Needs --verbose on the run, without which the capture holds only the result element.
+if (field === 'first_text') {
+  const texts = assistantTexts();
+  process.stdout.write(texts.length ? texts[0] : '');
+  process.exit(0);
+}
+
+const result = events.find((e) => e && e.type === 'result');
+if (!result) {
+  process.stderr.write(`extract: no element with type "result" in ${file}\n`);
+  process.exit(1);
+}
+
+let value = result[field];
+
+// A run stopped by --max-turns closes with subtype "error_max_turns" and no usable
+// `result`, even though assistant text was produced. Fall back to the text blocks of the
+// last assistant message so the case is scored on what the session actually said.
+if (field === 'result' && (value === undefined || value === null || value === 'undefined')) {
+  const texts = assistantTexts();
+  value = texts.length ? texts[texts.length - 1] : '';
+}
+
+process.stdout.write(value === undefined || value === null ? '' : String(value));

package/scripts/evals/run.sh

@@ -0,0 +1,369 @@
+#!/usr/bin/env bash
+# Behavioural eval harness for this package.
+#
+#   run.sh [--all | --case <id>...] [--reps N] [--model <id>] [--dry-run]
+#
+# Installs the package into a throwaway CLAUDE_CONFIG_DIR, runs each case against
+# a throwaway copy of a fixture repository with `claude -p`, and scores the answer
+# with the case's assert.sh. Nothing is written inside the git index of this repo:
+# results go to $LL_EVAL_RESULTS (default ~/.claude/ll-skills-evals)/<YYYY-MM-DD-HHMM>/ and the work
+# trees to a mktemp directory outside the repo, so the session under test never
+# discovers this project's own .claude/ or CLAUDE.md.
+#
+# Exit 0 when every selected case passed in at least min_pass reps (case.json,
+# capped at the number of reps actually run).
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO="$(cd "$HERE/../.." && pwd)"
+CASES_DIR="$HERE/cases"
+LIB="$HERE/lib"
+SHARED_FIXTURE="$REPO/scripts/fixtures/project"
+GIT_HISTORY="$REPO/scripts/fixtures/git-history.sh"
+PREAMBLE="$REPO/assets/preamble.md"
+INSTALLER="$REPO/bin/install.js"
+
+REPS=3
+MODEL=""
+DRY_RUN=0
+SELECTED=()
+
+die() { printf 'run.sh: %s\n' "$1" >&2; exit 2; }
+
+usage() {
+  sed -n '2,16p' "$HERE/run.sh" | sed 's/^# \{0,1\}//'
+  exit 0
+}
+
+# ---------------------------------------------------------------------------
+# arguments
+# ---------------------------------------------------------------------------
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --all)      SELECTED=(); shift ;;
+    --case)     [ $# -ge 2 ] || die "--case needs an id"; SELECTED+=("$2"); shift 2 ;;
+    --reps)     [ $# -ge 2 ] || die "--reps needs a number"; REPS="$2"; shift 2 ;;
+    --model)    [ $# -ge 2 ] || die "--model needs an id"; MODEL="$2"; shift 2 ;;
+    --dry-run)  DRY_RUN=1; shift ;;
+    -h|--help)  usage ;;
+    *)          die "unknown argument: $1 (use --help)" ;;
+  esac
+done
+
+case "$REPS" in (''|*[!0-9]*) die "--reps must be a positive integer" ;; esac
+[ "$REPS" -ge 1 ] || die "--reps must be at least 1"
+
+[ -d "$CASES_DIR" ] || die "no cases directory at $CASES_DIR"
+[ -f "$PREAMBLE" ]  || die "no preamble at $PREAMBLE"
+[ -f "$INSTALLER" ] || die "no installer at $INSTALLER"
+command -v node >/dev/null 2>&1 || die "node is required"
+command -v claude >/dev/null 2>&1 || [ "$DRY_RUN" -eq 1 ] || die "claude CLI is not on PATH"
+
+all_cases() {
+  local d
+  for d in "$CASES_DIR"/*/; do
+    [ -f "${d}case.json" ] || continue
+    basename "$d"
+  done
+}
+
+if [ "${#SELECTED[@]}" -eq 0 ]; then
+  mapfile -t SELECTED < <(all_cases)
+fi
+[ "${#SELECTED[@]}" -gt 0 ] || die "no cases selected"
+
+for id in "${SELECTED[@]}"; do
+  [ -f "$CASES_DIR/$id/case.json" ] || die "unknown case: $id"
+  [ -f "$CASES_DIR/$id/prompt.txt" ] || die "case $id has no prompt.txt"
+  [ -f "$CASES_DIR/$id/assert.sh" ] || die "case $id has no assert.sh"
+done
+
+# A case with "reuse" runs no claude call: it re-scores the work dir and the
+# capture of the case it names. Order the selection so the source runs first.
+reuse_of() { node -e '
+  const c = require(process.argv[1]);
+  process.stdout.write(c.reuse || "");
+' "$CASES_DIR/$1/case.json"; }
+
+field() { # field <case-id> <name> <default>
+  node -e '
+    const c = require(process.argv[1]);
+    const v = c[process.argv[2]];
+    process.stdout.write(v === undefined || v === null ? process.argv[3] : String(v));
+  ' "$CASES_DIR/$1/case.json" "$2" "$3"
+}
+
+ORDERED=()
+for id in "${SELECTED[@]}"; do [ -n "$(reuse_of "$id")" ] || ORDERED+=("$id"); done
+for id in "${SELECTED[@]}"; do [ -z "$(reuse_of "$id")" ] || ORDERED+=("$id"); done
+
+# ---------------------------------------------------------------------------
+# results directory (outside the git index) and work root (outside the repo)
+# ---------------------------------------------------------------------------
+
+STAMP="$(date +%Y-%m-%d-%H%M)"
+RESULTS="${LL_EVAL_RESULTS:-$HOME/.claude/ll-skills-evals}/$STAMP"
+mkdir -p "$RESULTS" || die "cannot create $RESULTS"
+
+TMP="$(mktemp -d "${TMPDIR:-/tmp}/ll-evals.XXXXXXXX")" || die "cannot create a work root"
+CONFIG="$TMP/config"
+
+printf 'results : %s\n' "$RESULTS"
+printf 'work    : %s\n' "$TMP"
+printf 'cases   : %s (reps %s)\n' "${#ORDERED[@]}" "$REPS"
+printf '\n'
+
+# ---------------------------------------------------------------------------
+# install the package into the throwaway config dir
+# ---------------------------------------------------------------------------
+
+install_cmd() {
+  printf 'CLAUDE_CONFIG_DIR=%s node %s --yes --no-settings' "$CONFIG" "$INSTALLER"
+}
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  printf '# install\n%s\n\n' "$(install_cmd)"
+else
+  mkdir -p "$CONFIG"
+  # The credentials live in the real config dir; a fresh CLAUDE_CONFIG_DIR has no auth
+  # of its own. A symlink, not a copy: a copy is a snapshot that goes stale as soon as
+  # the OAuth token is refreshed, and a long run then dies with "session expired".
+  # ANTHROPIC_API_KEY, when set, covers the same ground.
+  if [ -f "$HOME/.claude/.credentials.json" ] && [ ! -e "$CONFIG/.credentials.json" ]; then
+    ln -s "$HOME/.claude/.credentials.json" "$CONFIG/.credentials.json"
+  fi
+  if ! CLAUDE_CONFIG_DIR="$CONFIG" node "$INSTALLER" --yes --no-settings > "$RESULTS/install.log" 2>&1; then
+    cat "$RESULTS/install.log" >&2
+    die "the installer failed; see $RESULTS/install.log"
+  fi
+  printf 'installed into %s (%s skills)\n\n' "$CONFIG" "$(ls -1 "$CONFIG/skills" 2>/dev/null | wc -l)"
+fi
+
+# ---------------------------------------------------------------------------
+# one rep
+# ---------------------------------------------------------------------------
+
+prepare_workdir() { # prepare_workdir <case-id> <workdir> <history>
+  local id="$1" work="$2" history="$3" fixture="$CASES_DIR/$1/fixture"
+
+  if [ "$history" = "true" ]; then
+    # git-history.sh rebuilds <target> from the shared fixture and commits five times.
+    bash "$GIT_HISTORY" "$work" >/dev/null 2>&1 || return 1
+  else
+    [ -d "$fixture" ] || fixture="$SHARED_FIXTURE"
+    rm -rf "$work"; mkdir -p "$work"
+    cp -R "$fixture/." "$work/" 2>/dev/null || true
+    git -C "$work" init -q -b main
+    git -C "$work" add -A
+    git -C "$work" -c user.name=eval -c user.email=eval@example.com \
+      -c commit.gpgsign=false commit -q -m "chore: eval fixture" --allow-empty
+  fi
+
+  if [ -f "$CASES_DIR/$id/setup.sh" ]; then
+    bash "$CASES_DIR/$id/setup.sh" "$work" > "$work/.eval-setup.log" 2>&1 || return 1
+  fi
+
+  # The sha the run starts from, so an assert can look only at what the run added.
+  git -C "$work" rev-parse HEAD > "$work/.eval-base-sha" 2>/dev/null || echo "" > "$work/.eval-base-sha"
+  printf '.eval-base-sha\n.eval-setup.log\n' >> "$work/.git/info/exclude"
+  return 0
+}
+
+# prompt.txt may carry {{WORK}}, replaced with the absolute path of the work tree —
+# a brief passes absolute paths and no `cd`, and the work tree is created per rep.
+prompt_text() { # prompt_text <case-id> <workdir>
+  sed "s|{{WORK}}|$2|g" "$CASES_DIR/$1/prompt.txt"
+}
+
+claude_cmd() { # claude_cmd <case-id> <workdir> <max_turns> <agent> <permission_mode>
+  local id="$1" work="$2" turns="$3" agent="$4" perm="$5"
+  local cmd="env -u CLAUDECODE CLAUDE_CONFIG_DIR=$CONFIG claude -p \"\$(sed 's|{{WORK}}|$work|g' $CASES_DIR/$id/prompt.txt)\""
+  cmd="$cmd --max-turns $turns --output-format json"
+  cmd="$cmd --append-system-prompt \"\$(cat $PREAMBLE)\""
+  cmd="$cmd --permission-mode $perm --strict-mcp-config --verbose"
+  [ -n "$agent" ] && cmd="$cmd --agent $agent"
+  [ -n "$MODEL" ] && cmd="$cmd --model $MODEL"
+  printf '(cd %s && %s > %s/%s/rep1/out.json)' "$work" "$cmd" "$RESULTS" "$id"
+}
+
+run_claude() { # run_claude <case-id> <workdir> <turns> <agent> <perm> <out.json>
+  local id="$1" work="$2" turns="$3" agent="$4" perm="$5" outjson="$6"
+  local args=(-p "$(prompt_text "$id" "$work")"
+              --max-turns "$turns"
+              --output-format json
+              --append-system-prompt "$(cat "$PREAMBLE")"
+              --permission-mode "$perm"
+              --strict-mcp-config
+              --verbose)
+  [ -n "$agent" ] && args+=(--agent "$agent")
+  [ -n "$MODEL" ] && args+=(--model "$MODEL")
+  ( cd "$work" && env -u CLAUDECODE CLAUDE_CONFIG_DIR="$CONFIG" claude "${args[@]}" ) \
+    > "$outjson" 2>"${outjson%.json}.stderr"
+}
+
+# ---------------------------------------------------------------------------
+# the loop
+# ---------------------------------------------------------------------------
+
+ROWS=()          # "case|rep|PASS/FAIL|cost|duration|turns|first failure"
+declare -A PASSES=() MINPASS=()
+TOTAL_COST=0
+
+for id in "${ORDERED[@]}"; do
+  MAX_TURNS="$(field "$id" max_turns 20)"
+  HISTORY="$(field "$id" history false)"
+  MIN_PASS="$(field "$id" min_pass 2)"
+  AGENT="$(field "$id" agent '')"
+  PERM="$(field "$id" permission_mode bypassPermissions)"
+  REUSE="$(reuse_of "$id")"
+  [ "$AGENT" = "null" ] && AGENT=""
+  MINPASS["$id"]="$MIN_PASS"
+  PASSES["$id"]=0
+
+  if [ "$DRY_RUN" -eq 1 ]; then
+    work="$TMP/work-$id-1"
+    printf '# case %s  (max_turns %s · history %s · min_pass %s%s)\n' \
+      "$id" "$MAX_TURNS" "$HISTORY" "$MIN_PASS" "${AGENT:+ · agent $AGENT}"
+    if [ -n "$REUSE" ]; then
+      printf '# no claude call: re-scores the work dir and capture of %s\n' "$REUSE"
+      printf 'bash %s/assert.sh %s %s %s\n\n' \
+        "$CASES_DIR/$id" "$TMP/work-$REUSE-1" "$RESULTS/$REUSE/rep1/out.json" "$RESULTS/$REUSE/rep1/out.txt"
+      continue
+    fi
+    if [ "$HISTORY" = "true" ]; then
+      printf 'bash %s %s\n' "$GIT_HISTORY" "$work"
+    else
+      src="$CASES_DIR/$id/fixture"; [ -d "$src" ] || src="$SHARED_FIXTURE"
+      printf 'cp -R %s/. %s/ && git -C %s init -q -b main && git -C %s commit -m "chore: eval fixture"\n' \
+        "$src" "$work" "$work" "$work"
+    fi
+    [ -f "$CASES_DIR/$id/setup.sh" ] && printf 'bash %s/setup.sh %s\n' "$CASES_DIR/$id" "$work"
+    rep1="$RESULTS/$id/rep1"
+    printf '%s\n' "$(claude_cmd "$id" "$work" "$MAX_TURNS" "$AGENT" "$PERM")"
+    printf 'node %s/extract.js %s/out.json result > %s/out.txt\n' "$LIB" "$rep1" "$rep1"
+    printf 'bash %s/assert.sh %s %s/out.json %s/out.txt\n\n' "$CASES_DIR/$id" "$work" "$rep1" "$rep1"
+    continue
+  fi
+
+  for rep in $(seq 1 "$REPS"); do
+    repdir="$RESULTS/$id/rep$rep"
+    mkdir -p "$repdir"
+    outjson="$repdir/out.json"
+    outtxt="$repdir/out.txt"
+    cost="-" ; dur="-" ; turns="-" ; firstfail="" ; verdict="FAIL"
+
+    if [ -n "$REUSE" ]; then
+      work="$TMP/work-$REUSE-$rep"
+      src="$RESULTS/$REUSE/rep$rep"
+      if [ ! -d "$work" ] || [ ! -f "$src/out.json" ]; then
+        firstfail="source case $REUSE was not run in this invocation"
+        ROWS+=("$id|$rep|FAIL|-|-|-|$firstfail")
+        printf '  %-26s rep %s  FAIL  (%s)\n' "$id" "$rep" "$firstfail"
+        continue
+      fi
+      cp "$src/out.json" "$outjson"; cp "$src/out.txt" "$outtxt"
+    else
+      work="$TMP/work-$id-$rep"
+      if ! prepare_workdir "$id" "$work" "$HISTORY"; then
+        firstfail="fixture preparation failed"
+        ROWS+=("$id|$rep|FAIL|-|-|-|$firstfail")
+        printf '  %-26s rep %s  FAIL  (%s)\n' "$id" "$rep" "$firstfail"
+        continue
+      fi
+      run_claude "$id" "$work" "$MAX_TURNS" "$AGENT" "$PERM" "$outjson"
+      if ! node "$LIB/extract.js" "$outjson" result > "$outtxt" 2>"$repdir/extract.err"; then
+        firstfail="no result element in out.json ($(head -c 120 "$repdir/extract.err" | tr '\n' ' '))"
+        ROWS+=("$id|$rep|FAIL|-|-|-|$firstfail")
+        printf '  %-26s rep %s  FAIL  (%s)\n' "$id" "$rep" "$firstfail"
+        continue
+      fi
+    fi
+
+    cost="$(node "$LIB/extract.js" "$outjson" total_cost_usd 2>/dev/null)"; [ -n "$cost" ] || cost="-"
+    ms="$(node "$LIB/extract.js" "$outjson" duration_ms 2>/dev/null)"
+    turns="$(node "$LIB/extract.js" "$outjson" num_turns 2>/dev/null)"; [ -n "$turns" ] || turns="-"
+    if [ -n "$ms" ]; then dur="$(node -e 'process.stdout.write((Number(process.argv[1])/1000).toFixed(1))' "$ms")"; fi
+    if [ "$cost" != "-" ]; then
+      TOTAL_COST="$(node -e 'process.stdout.write((Number(process.argv[1])+Number(process.argv[2])).toFixed(4))' "$TOTAL_COST" "$cost")"
+      cost="$(node -e 'process.stdout.write(Number(process.argv[1]).toFixed(4))' "$cost")"
+    fi
+
+    bash "$CASES_DIR/$id/assert.sh" "$work" "$outjson" "$outtxt" > "$repdir/assert.log" 2>&1
+    arc=$?
+    if [ "$arc" -eq 0 ]; then
+      verdict="PASS"
+      PASSES["$id"]=$(( ${PASSES["$id"]} + 1 ))
+    else
+      firstfail="$(grep -m1 '^FAIL: ' "$repdir/assert.log" | sed 's/^FAIL: //')"
+      [ -n "$firstfail" ] || firstfail="assert.sh exited non-zero with no FAIL line"
+      # A run cut short by the turn cap is a budget problem, not a behavioural one: say so.
+      sub="$(node "$LIB/extract.js" "$outjson" subtype 2>/dev/null)"
+      [ "$sub" = "success" ] || [ -z "$sub" ] || firstfail="[$sub] $firstfail"
+    fi
+
+    ROWS+=("$id|$rep|$verdict|$cost|$dur|$turns|$firstfail")
+    printf '  %-26s rep %s  %s  cost %s  %ss  turns %s%s\n' \
+      "$id" "$rep" "$verdict" "$cost" "$dur" "$turns" "${firstfail:+  — $firstfail}"
+  done
+done
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  printf '# dry run: %s case blocks printed, no claude call made\n' "${#ORDERED[@]}"
+  exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# RESULTS.md and summary.json
+# ---------------------------------------------------------------------------
+
+EXIT=0
+{
+  printf '# Eval results — %s\n\n' "$STAMP"
+  printf 'model: %s · reps: %s · cases: %s · total cost: USD %s\n\n' \
+    "${MODEL:-default}" "$REPS" "${#ORDERED[@]}" "$TOTAL_COST"
+  printf '| case | rep | verdict | cost USD | duration s | turns | first assert failure |\n'
+  printf '|---|---|---|---|---|---|---|\n'
+  for row in "${ROWS[@]}"; do
+    IFS='|' read -r c r v co du tu ff <<< "$row"
+    printf '| %s | %s | %s | %s | %s | %s | %s |\n' "$c" "$r" "$v" "$co" "$du" "$tu" "${ff:-—}"
+  done
+  printf '\n## Case verdicts\n\n'
+  printf '| case | passed | of reps | min_pass | verdict |\n|---|---|---|---|---|\n'
+  for id in "${ORDERED[@]}"; do
+    need="${MINPASS[$id]}"
+    [ "$need" -le "$REPS" ] || need="$REPS"
+    got="${PASSES[$id]}"
+    if [ "$got" -ge "$need" ]; then v=PASS; else v=FAIL; EXIT=1; fi
+    printf '| %s | %s | %s | %s | %s |\n' "$id" "$got" "$REPS" "$need" "$v"
+  done
+  printf '\nWork trees kept at `%s`.\n' "$TMP"
+} > "$RESULTS/RESULTS.md"
+
+node - "$RESULTS/summary.json" "$STAMP" "${MODEL:-default}" "$REPS" "$TMP" "$RESULTS" "$TOTAL_COST" "$EXIT" <<'NODE' "${ROWS[@]}"
+const fs = require('fs');
+const [out, stamp, model, reps, tmp, results, cost, exitCode, ...rows] = process.argv.slice(2);
+const byCase = {};
+for (const row of rows) {
+  const [id, rep, verdict, c, d, t, ff] = row.split('|');
+  (byCase[id] = byCase[id] || []).push({
+    rep: Number(rep), pass: verdict === 'PASS',
+    cost_usd: c === '-' ? null : Number(c),
+    duration_s: d === '-' ? null : Number(d),
+    turns: t === '-' ? null : Number(t),
+    first_failure: ff || null,
+  });
+}
+fs.writeFileSync(out, JSON.stringify({
+  stamp, model, reps: Number(reps), results_dir: results, work_dir: tmp,
+  total_cost_usd: Number(cost), exit_code: Number(exitCode),
+  cases: Object.entries(byCase).map(([id, r]) => ({
+    id, passed: r.filter((x) => x.pass).length, reps: r,
+  })),
+}, null, 2) + '\n');
+NODE
+
+printf '\n%s\n' "$RESULTS/RESULTS.md"
+printf 'total cost USD %s · exit %s\n' "$TOTAL_COST" "$EXIT"
+exit "$EXIT"

package/scripts/fixtures/auto-closed/PLAN.md

@@ -0,0 +1,5 @@
+# PLAN — fixture delivered
+
+## §8 Phases
+
+See ROADMAP.md. Current phase: 02.

package/scripts/fixtures/auto-closed/PROGRESS.md

@@ -0,0 +1,20 @@
+# PROGRESS — fixture delivered
+
+<!-- ll-state -->
+phase: 02
+milestones:
+  M1: { passes: true, commit: a1b2c3d, accepted_at: 2026-09-08T10:00:00Z }
+  M2: { passes: true, commit: b2c3d4e, accepted_at: 2026-09-08T15:30:00Z }
+<!-- /ll-state -->
+
+## Epilogue — phase 01 — 2026-09-07
+
+passed: M1 · left: none
+milestones passed 1/1 · questions asked 0 / assumptions 0 / band-1 open 0 · amendments 0 · verification: none
+▶ Next — `/clear`, then `ll-implement 2`
+
+## Epilogue — phase 02 — 2026-09-08
+
+passed: M1, M2 · left: none
+milestones passed 2/2 · questions asked 0 / assumptions 0 / band-1 open 0 · amendments 0 · verification: none
+▶ Next — `/clear`, then `ll-close`

package/scripts/fixtures/auto-closed/ROADMAP.md

@@ -0,0 +1,6 @@
+# ROADMAP — fixture delivered
+
+| phase | name | depends_on | requirements | state |
+|---|---|---|---|---|
+| 01 | intake | — | REQ-a | DONE (docs/history/v1.0) |
+| 02 | reporting | 01 | REQ-b | DONE (docs/history/v1.0) |

package/scripts/fixtures/auto-closed/docs/DELIVERY.md

@@ -0,0 +1,3 @@
+# DELIVERY — fixture delivered
+
+Delivered on 2026-09-08: phases 01 and 02 closed, every criterion green.