ll-skills 2.0.2 → 3.1.0

package/scripts/evals/cases/router-no-skill/fixture/decisions/README.md

@@ -0,0 +1,3 @@
+# decisions
+
+One file per recorded decision, `DEC-NNNN-<slug>.md`. Empty until the first one is written.

package/scripts/evals/cases/router-no-skill/prompt.txt

@@ -0,0 +1 @@
+pesquise as opções de fila para Node e me diz qual você usaria num projeto pequeno

package/scripts/evals/cases/router-small/assert.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# SMALL: verb + addressable target -> answer with a number, write nothing, start no skill.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$OUT_TXT" '[0-9]+' 'the answer carries a number'
+
+dirty="$(git -C "$WORK" status --porcelain)"
+if [ -z "$dirty" ]; then
+  ok 'git status --porcelain is empty: nothing was written'
+else
+  fail "the working tree was changed: $(printf '%s' "$dirty" | tr '\n' ' ')"
+fi
+
+no_path "$WORK/PROGRESS.md" 'no PROGRESS.md was created'
+no_path "$WORK/phases"      'no phases/ was created'
+
+no_tool_use "$OUT_JSON" Skill 'no Skill tool call anywhere in the capture'
+
+finish

package/scripts/evals/cases/router-small/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 6,
+  "history": false,
+  "min_pass": 2,
+  "agent": null,
+  "permission_mode": "acceptEdits",
+  "note": "Own fixture, deliberately without PROGRESS.md: the RESUME rule fires on the first turn in a repo that has one, and would compete with SMALL. The session answers directly, writes nothing and starts no skill."
+}

package/scripts/evals/cases/router-small/fixture/README.md

@@ -0,0 +1,17 @@
+# Ledger service
+
+A service that reconciles provider events against the ledger.
+
+## Install
+
+    npm install
+
+## Run
+
+    npm start
+
+## Test
+
+    npm test
+
+The reconciliation job runs every hour.

package/scripts/evals/cases/router-small/prompt.txt

@@ -0,0 +1 @@
+conta as linhas de README.md

package/scripts/evals/cases/scout-no-plan/assert.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# The scout writes CODE-CONTEXT.md within its cap and never opens the project PLAN.md.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+CTX="$WORK/phases/07/CODE-CONTEXT.md"
+
+if [ -f "$CTX" ]; then
+  n="$(wc -l < "$CTX")"
+  ok "phases/07/CODE-CONTEXT.md exists ($n lines)"
+  if [ "$n" -le 120 ]; then ok "the file is within its 120-line cap"; else fail "CODE-CONTEXT.md has $n lines, cap is 120"; fi
+else
+  fail 'phases/07/CODE-CONTEXT.md was not written'
+fi
+
+# No Read tool_use whose file_path is the project PLAN.md at the root of the work tree.
+offenders="$(node -e '
+  const fs = require("fs");
+  const work = process.argv[2];
+  let data; try { data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); } catch { process.exit(0); }
+  const events = Array.isArray(data) ? data : [data];
+  const hits = [];
+  for (const ev of events) {
+    const content = ev && ev.message && ev.message.content;
+    if (!Array.isArray(content)) continue;
+    for (const b of content) {
+      if (!b || b.type !== "tool_use" || b.name !== "Read") continue;
+      const p = (b.input && b.input.file_path) || "";
+      if (p === work + "/PLAN.md" || p === "PLAN.md" || p === "./PLAN.md") hits.push(p);
+    }
+  }
+  process.stdout.write(hits.join(" "));
+' "$OUT_JSON" "$WORK")"
+
+if [ -z "$offenders" ]; then
+  ok 'no Read of the project PLAN.md at the work-tree root'
+else
+  fail "the scout read the project PLAN.md: $offenders"
+fi
+
+finish

package/scripts/evals/cases/scout-no-plan/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 30,
+  "history": true,
+  "min_pass": 2,
+  "agent": "ll-scout",
+  "permission_mode": "bypassPermissions",
+  "note": "The scout must not read the project PLAN.md at the repository root. The phase plan is not scored here."
+}

package/scripts/evals/cases/scout-no-plan/prompt.txt

@@ -0,0 +1,8 @@
+PHASE         07 — every provider event lands in `billing_events` exactly once and mismatches are visible   (only this phase)
+MODEL         sonnet / medium — always, contract phases included
+FILES         create: {{WORK}}/src/ingest.ts · change: {{WORK}}/src/pay.ts
+SYMBOLS       none
+OUTPUT        {{WORK}}/phases/07/CODE-CONTEXT.md — at most 120 lines, Write not heredoc
+DO NOT        read the project PLAN.md, PROGRESS.md, phases/*/PLAN.md or decisions/; propose a plan;
+              run tests or builds
+RETURN        your fixed block, at most 10 lines: path, files classified, analogs, readers, assumed values

package/scripts/evals/cases/verifier-weakened-test/assert.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# A test whose assertion was removed without a DEC must not ride in an approval.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../lib" && pwd)/assert.sh"
+
+WORK="$1"; OUT_JSON="$2"; OUT_TXT="$3"
+
+contains "$OUT_TXT" 'REJECTED'       'the return carries the verdict REJECTED'
+contains "$OUT_TXT" 'test/a\.test\.ts' 'the return names test/a.test.ts'
+
+if [ -f "$WORK/phases/07/VERIFICATION.md" ]; then
+  ok 'phases/07/VERIFICATION.md was written'
+else
+  fail 'phases/07/VERIFICATION.md was not written at the path the brief gave'
+fi
+
+# The verifier fixes nothing: the weakened test is still weakened.
+absent "$WORK/test/a.test.ts" 'expect\(' 'the verifier did not repair the test it flagged'
+
+finish

package/scripts/evals/cases/verifier-weakened-test/case.json

@@ -0,0 +1,8 @@
+{
+  "max_turns": 40,
+  "history": true,
+  "min_pass": 2,
+  "agent": "ll-verifier",
+  "permission_mode": "bypassPermissions",
+  "note": "setup.sh commits a feat(M2) that strips the only expect() from test/a.test.ts, with no DEC id anywhere. The process check must catch it."
+}

package/scripts/evals/cases/verifier-weakened-test/prompt.txt

@@ -0,0 +1,13 @@
+MODE          phase
+ROADMAP       {{WORK}}/ROADMAP.md — section "Phase 07"
+PLAN          {{WORK}}/phases/07/PLAN.md — truths:, milestones, acceptance:, verification:, ## Errata
+DECISIONS     {{WORK}}/decisions/DEC-0041-cents.md
+CRITERIA      SC-01 The reconciliation job ingests a provider batch and writes one row per event.
+              SC-02 A replayed batch produces no duplicate rows.
+              SC-03 Cent-level mismatches are reported instead of silently dropped.
+              SC-04 The operator can list tolerated rows for a day.
+SLICE         branch main · the whole history · worktree {{WORK}}
+OUTPUT        {{WORK}}/phases/07/VERIFICATION.md — Write, not heredoc
+DO NOT        open PROGRESS.md before every state is written; edit code, tests, plan or passes;
+              run the whole suite to prove one criterion
+RETURN        your fixed block, at most 20 lines: verdict, product/process, state counts, BLOCKS, gaps

package/scripts/evals/cases/verifier-weakened-test/setup.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Commits a feat(M2) that weakens test/a.test.ts: the only expect() is dropped and the
+# test becomes unconditional. No DEC id in the message and none under decisions/.
+set -euo pipefail
+
+WORK="${1:?usage: setup.sh <workdir>}"
+
+cat > "$WORK/test/a.test.ts" <<'TS'
+import { reconcile } from '../src/a';
+
+test('reconcile dedupes a replayed batch', () => {
+  reconcile(['e1', 'e1', 'e2']);
+});
+TS
+
+git -C "$WORK" add test/a.test.ts
+GIT_AUTHOR_DATE="2026-09-10T10:00:00+00:00" GIT_COMMITTER_DATE="2026-09-10T10:00:00+00:00" \
+  git -C "$WORK" -c user.name=fixture -c user.email=fixture@example.com -c commit.gpgsign=false \
+  commit -q -m "feat(M2): simplify the reconcile assertion"

package/scripts/evals/fixtures/manual-contract/out.json

@@ -0,0 +1,29 @@
+[
+  {
+    "type": "system",
+    "subtype": "init",
+    "session_id": "manual-contract-fixture",
+    "model": "claude-sonnet-5"
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+        }
+      ]
+    }
+  },
+  {
+    "type": "result",
+    "subtype": "success",
+    "is_error": false,
+    "duration_ms": 1820,
+    "num_turns": 1,
+    "total_cost_usd": 0.0091,
+    "result": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+  }
+]

package/scripts/evals/fixtures/manual-contract/out.txt

@@ -0,0 +1,5 @@
+Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:
+
+/ll-research compare authentication providers for the signup flow
+
+▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`

package/scripts/evals/fixtures/manual-contract/with-skill.json

@@ -0,0 +1,46 @@
+[
+  {
+    "type": "system",
+    "subtype": "init",
+    "session_id": "manual-contract-fixture-with-skill",
+    "model": "claude-sonnet-5"
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+        }
+      ]
+    }
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "tool_use",
+          "id": "toolu_01manualcontract",
+          "name": "Skill",
+          "input": {
+            "skill": "ll-research",
+            "args": "compare authentication providers for the signup flow"
+          }
+        }
+      ]
+    }
+  },
+  {
+    "type": "result",
+    "subtype": "success",
+    "is_error": false,
+    "duration_ms": 2140,
+    "num_turns": 2,
+    "total_cost_usd": 0.0134,
+    "result": "Comparing authentication providers for the signup flow is a research task, not something I can answer from what is already in the repo. Paste this to start it:\n\n/ll-research compare authentication providers for the signup flow\n\n▶ Next — `/clear` then `/ll-research compare authentication providers for the signup flow`"
+  }
+]

package/scripts/evals/fixtures/router-no-skill/fail.txt

@@ -0,0 +1,5 @@
+Isso é trabalho do skill de pesquisa: eu não inicio um skill por conta própria. Cole o comando `/ll-research`:
+
+    `/ll-research opções de fila para Node`
+
+O skill compara as opções, confere as fontes e escreve `docs/research/RESEARCH.md`.

package/scripts/evals/fixtures/router-no-skill/out.json

@@ -0,0 +1,27 @@
+[
+  {
+    "type": "system",
+    "subtype": "init",
+    "session_id": "router-no-skill-fixture",
+    "model": "claude-sonnet-5"
+  },
+  {
+    "type": "assistant",
+    "message": {
+      "role": "assistant",
+      "content": [
+        {
+          "type": "text",
+          "text": "Para um projeto pequeno em Node eu usaria `bullmq` sobre `redis`: fila com retry e dead-letter prontos.\nAs outras que olhei: `pg-boss` no Postgres, `rabbitmq` e `sqs`."
+        }
+      ]
+    }
+  },
+  {
+    "type": "result",
+    "subtype": "success",
+    "is_error": false,
+    "num_turns": 3,
+    "result": "Para um projeto pequeno em Node eu usaria `bullmq` sobre `redis`: fila com retry e dead-letter prontos.\nAs outras que olhei: `pg-boss` no Postgres, `rabbitmq` e `sqs`."
+  }
+]

package/scripts/evals/fixtures/router-no-skill/pass.txt

@@ -0,0 +1,4 @@
+Para um projeto pequeno em Node eu usaria `bullmq` sobre `redis`: fila durável, retry, backoff e dead-letter já vêm prontos.
+As outras que valem olhar: `pg-boss` (a fila mora no Postgres — uma dependência a menos quando o banco já existe), `rabbitmq` (roteamento rico, mas é mais um serviço para operar) e `sqs` (gerenciado, com o preço de amarrar o projeto à AWS).
+Regra prática: enquanto tudo couber num processo e o Postgres já estiver de pé, `pg-boss`; quando precisar de concorrência, rate limit e repetição por fila, `bullmq`.
+Nada disso foi medido aqui: é leitura de documentação, não `benchmark` rodado por mim.

package/scripts/evals/lib/assert.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Sourced by every cases/<id>/assert.sh.
+#
+# Contract of an assert script:
+#   bash assert.sh <workdir> <out.json> <out.txt>   -> exit 0 pass, 1 fail
+# It prints one line per check: "ok: <what>" or "FAIL: <what>".
+# run.sh reads the first FAIL line into the results table.
+#
+# Provided helpers:
+#   ok <msg>                     record a passed check
+#   fail <msg>                   record a failed check
+#   check <cond-exit> <msg>      record from an exit code already computed
+#   contains <file> <regex> <msg>    grep -Eq
+#   absent <file> <regex> <msg>      grep -Eq must not match
+#   no_path <path> <msg>         path must not exist
+#   no_tool_use <out.json> <tool-name> <msg>   no tool_use block named <tool-name> anywhere
+#   finish                       exit with the verdict
+
+EVAL_FAILURES=0
+
+ok()   { printf 'ok: %s\n' "$1"; }
+fail() { printf 'FAIL: %s\n' "$1"; EVAL_FAILURES=$((EVAL_FAILURES + 1)); }
+
+check() { # check <exit-code> <msg>
+  if [ "$1" -eq 0 ]; then ok "$2"; else fail "$2"; fi
+}
+
+contains() { # contains <file> <extended-regex> <msg>
+  if [ -f "$1" ] && grep -Eq -- "$2" "$1"; then ok "$3"; else fail "$3"; fi
+}
+
+absent() { # absent <file> <extended-regex> <msg>
+  if [ ! -f "$1" ] || ! grep -Eq -- "$2" "$1"; then ok "$3"; else fail "$3"; fi
+}
+
+no_path() { # no_path <path> <msg>
+  if [ ! -e "$1" ]; then ok "$2"; else fail "$2"; fi
+}
+
+first_line() { # first_line <file> -> first non-empty line
+  [ -f "$1" ] || return 0
+  grep -m1 -v '^[[:space:]]*$' "$1" 2>/dev/null || true
+}
+
+# The regime is stated "before doing anything": it belongs to the first assistant
+# message of the run, not to the final answer. This reads it out of the capture.
+first_text() { # first_text <out.json> -> the whole first assistant message
+  node "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/extract.js" "$1" first_text 2>/dev/null || true
+}
+
+first_text_line() { # first_text_line <out.json> -> first non-empty line of the first assistant text
+  first_text "$1" | grep -m1 -v '^[[:space:]]*$' || true
+}
+
+first_text_contains() { # first_text_contains <out.json> <extended-regex> <msg>
+  if first_text "$1" | grep -Eq -- "$2"; then ok "$3"; else fail "$3"; fi
+}
+
+# no_tool_use <out.json> <tool-name> <msg>
+# Scans every assistant event's message.content for a tool_use block whose
+# `name` equals <tool-name>. Passes when none is found; fails naming the
+# first hit (the manual contract: no skill is started by a tool call).
+# A capture that is missing or is not readable JSON proves nothing: it fails
+# (B-003/B-020 — a rep whose out.json never landed used to score PASS).
+no_tool_use() {
+  local file="$1" name="$2" msg="$3" hit rc
+  if [ ! -f "$file" ]; then
+    fail "$msg: no capture at $file"
+    return
+  fi
+  hit="$(node -e '
+    const fs = require("fs");
+    let data;
+    try { data = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); }
+    catch { process.exit(3); }
+    const events = Array.isArray(data) ? data : [data];
+    const wanted = process.argv[2];
+    for (const ev of events) {
+      if (!ev || ev.type !== "assistant") continue;
+      const content = ev.message && ev.message.content;
+      if (!Array.isArray(content)) continue;
+      for (const b of content) {
+        if (b && b.type === "tool_use" && b.name === wanted) {
+          process.stdout.write(b.name);
+          process.exit(0);
+        }
+      }
+    }
+  ' "$file" "$name" 2>/dev/null)"
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    fail "$msg: the capture at $file is not readable JSON"
+  elif [ -z "$hit" ]; then
+    ok "$msg"
+  else
+    fail "$msg: found a $hit tool_use call"
+  fi
+}
+
+base_sha() { # base_sha <workdir> -> the HEAD recorded before the run, or empty
+  [ -f "$1/.eval-base-sha" ] && cat "$1/.eval-base-sha"
+}
+
+finish() {
+  if [ "$EVAL_FAILURES" -eq 0 ]; then exit 0; fi
+  exit 1
+}

package/scripts/evals/lib/extract.js

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+'use strict';
+
+// Reads one field of the `type: "result"` element of a `claude --output-format json`
+// capture. The capture is either a single object or an array of events
+// (system/init … assistant … result); only the result element carries the totals.
+//
+//   node extract.js <out.json> [field]     field defaults to "result"
+//
+// Exits 1 when the file is not JSON or has no result element, so the caller can
+// tell "the run produced nothing" from "the run produced an empty answer".
+
+const fs = require('fs');
+
+const file = process.argv[2];
+const field = process.argv[3] || 'result';
+
+let raw;
+try {
+  raw = fs.readFileSync(file, 'utf8');
+} catch (e) {
+  process.stderr.write(`extract: cannot read ${file}: ${e.message}\n`);
+  process.exit(1);
+}
+
+let data;
+try {
+  data = JSON.parse(raw);
+} catch {
+  process.stderr.write(`extract: ${file} is not valid JSON\n`);
+  process.exit(1);
+}
+
+const events = Array.isArray(data) ? data : [data];
+
+const assistantTexts = () => {
+  const out = [];
+  for (const ev of events) {
+    if (!ev || ev.type !== 'assistant') continue;
+    const content = ev.message && ev.message.content;
+    if (!Array.isArray(content)) continue;
+    const t = content.filter((b) => b && b.type === 'text').map((b) => b.text).join('\n');
+    if (t.trim()) out.push(t);
+  }
+  return out;
+};
+
+// Pseudo-field: the first non-empty assistant message. The regime line is stated
+// "before doing anything", so it lives in the first turn, not in the final result.
+// Needs --verbose on the run, without which the capture holds only the result element.
+if (field === 'first_text') {
+  const texts = assistantTexts();
+  process.stdout.write(texts.length ? texts[0] : '');
+  process.exit(0);
+}
+
+const result = events.find((e) => e && e.type === 'result');
+if (!result) {
+  process.stderr.write(`extract: no element with type "result" in ${file}\n`);
+  process.exit(1);
+}
+
+let value = result[field];
+
+// A run stopped by --max-turns closes with subtype "error_max_turns" and no usable
+// `result`, even though assistant text was produced. Fall back to the text blocks of the
+// last assistant message so the case is scored on what the session actually said.
+if (field === 'result' && (value === undefined || value === null || value === 'undefined')) {
+  const texts = assistantTexts();
+  value = texts.length ? texts[texts.length - 1] : '';
+}
+
+process.stdout.write(value === undefined || value === null ? '' : String(value));