@iceinvein/agent-skills 0.22.0 → 0.23.0

package/skills/magpie/evals/local-mode-hands-back/fixture.sh

@@ -0,0 +1,300 @@
+#!/usr/bin/env bash
+# A local review of the workspace's own branch, done through peer review. The
+# local ladder ends at handoff: print the findings to the caller, archive the
+# run, end the turn. There is no PR, so nothing is served, rendered or posted.
+#
+# The workspace is the repo under review, because local mode reviews the
+# caller's working directory in place. The run directory sits under the
+# workspace rather than ~/.magpie because file graders refuse to follow a link
+# out of the workspace. `magpie --list-runs` is what names the path a resume
+# uses, so the shim reports this one.
+set -euo pipefail
+
+BRANCH="feat/tenant-settings-cache"
+RUN_ID="local-feat-tenant-settings-cache-1789600000"
+RUN_DIR="$PWD/runs/$RUN_ID"
+ARCHIVE_DIR="$PWD/archive"
+CALLS="$PWD/.magpie-calls.log"
+
+mkdir -p "$RUN_DIR"/findings "$HOME/shims"
+: > "$CALLS"
+
+cat > "$HOME/shim-config" <<EOF
+RUN_ID="$RUN_ID"
+RUN_DIR="$RUN_DIR"
+ARCHIVE_DIR="$ARCHIVE_DIR"
+CALLS="$CALLS"
+EOF
+
+# The real magpie and gh are outside the eval sandbox and cannot be executed
+# from inside it, so the child gets fakes rather than exit 126. The gh fake is
+# here to catch a post that goes around the CLI: a local run has no PR at all.
+mkdir -p "$HOME/tmp"
+cat > "$HOME/.zshenv" <<'RC'
+export PATH="$HOME/shims:/usr/bin:/bin:/usr/sbin:/sbin"
+# /usr/bin/python3 is the Xcode shim, and without a writable TMPDIR it fails
+# trying to create its xcrun cache in a directory the sandbox blocks.
+export TMPDIR="$HOME/tmp"
+RC
+
+# Unlike the PR cases' fakes, status reads target.json and picks the ladder the
+# way scripts/status-cmd.ts does, so this run resumes at handoff, not report.
+cat > "$HOME/shims/magpie" <<'SHIM'
+#!/usr/bin/env bash
+. "$HOME/shim-config"
+echo "magpie $*" >> "$CALLS"
+case "${1:-}" in
+  --list-runs)
+    if [ -d "$RUN_DIR" ]; then
+      printf '%s\tactive\t%s\n' "$RUN_ID" "$RUN_DIR"
+    else
+      printf '%s\tarchived\t%s\n' "$RUN_ID" "$ARCHIVE_DIR/$RUN_ID"
+    fi
+    ;;
+  status)
+    python3 - "${2:-$RUN_DIR}" <<'STATUS'
+import json, pathlib, sys
+
+run = pathlib.Path(sys.argv[1])
+PR = ['setup', 'context', 'specialists', 'dedupe', 'critic', 'peer-review', 'report', 'post']
+LOCAL = ['setup', 'context', 'specialists', 'dedupe', 'critic', 'peer-review', 'handoff']
+target = run / 'target.json'
+mode = json.loads(target.read_text())['mode'] if target.exists() else 'pr'
+ORDER = LOCAL if mode == 'local' else PR
+last, error = None, None
+for line in (run / 'log.jsonl').read_text().splitlines():
+    if not line.strip():
+        continue
+    try:
+        entry = json.loads(line)
+    except ValueError:
+        continue
+    if entry.get('status') == 'error':
+        error = entry.get('stage')
+        break
+    if entry.get('status') in ('done', 'skipped') and entry.get('stage') in ORDER:
+        last = entry['stage']
+index = ORDER.index(last) + 1 if last else 0
+print(json.dumps({'lastCompleted': last, 'next': ORDER[index] if index < len(ORDER) else 'cleanup', 'error': error}))
+STATUS
+    ;;
+  target)
+    python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["root"])' "${2:-$RUN_DIR}/target.json"
+    ;;
+  handoff)
+    python3 - "${2:-$RUN_DIR}" <<'HANDOFF'
+import json, pathlib, sys, time
+
+run = pathlib.Path(sys.argv[1])
+if json.loads((run / 'target.json').read_text())['mode'] != 'local':
+    sys.stderr.write(f'magpie: handoff only applies to local runs; {run} is a PR run\n')
+    sys.exit(2)
+for name in ('findings.final.json', 'local.json'):
+    if not (run / name).exists():
+        sys.stderr.write(f'magpie: handoff: {run / name} is missing\n')
+        sys.exit(4)
+findings = json.loads((run / 'findings.final.json').read_text())
+meta = json.loads((run / 'local.json').read_text())
+subject = 'detached HEAD' if meta['branch'] is None else f"local branch {meta['branch']}"
+# The labels the fixture's findings use, bolded as formatFindingDescriptionMarkdown does.
+LABELS = ('Observation', 'Why it matters', 'Suggested direction')
+
+
+def markdown(description):
+    paragraphs = description.split('\n\n')
+    for label in LABELS:
+        paragraphs = [f'**{label}:**' + p[len(label) + 1:] if p.startswith(f'{label}:') else p for p in paragraphs]
+    return '\n\n'.join(paragraphs)
+
+
+rank = {'must-fix': 0, 'should-fix': 1}
+actionable = sorted((f for f in findings if f['risk']['action'] in rank), key=lambda f: rank[f['risk']['action']])
+optional = [f for f in findings if f['risk']['action'] not in rank]
+counts = {a: sum(f['risk']['action'] == a for f in findings) for a in rank}
+sections = [
+    f"# magpie local review: {subject} against {meta['baseRef']}",
+    f"{counts['must-fix']} must-fix, {counts['should-fix']} should-fix, {len(optional)} optional",
+]
+for n, f in enumerate(actionable + optional, 1):
+    anchor = f['file'] if f.get('line') is None else f"{f['file']}:{f['line']}"
+    sections.append(f"{n}. [{f['risk']['action']}] {anchor}: {f['title']}\n\n{markdown(f['description'])}")
+sections.append('Address every must-fix and should-fix, or say why not. The rest are optional.')
+print('\n\n'.join(sections))
+with (run / 'log.jsonl').open('a') as log:
+    log.write(json.dumps({'stage': 'handoff', 'status': 'done', 'findings': len(findings), 'ts': int(time.time() * 1000)}) + '\n')
+HANDOFF
+    ;;
+  cleanup)
+    run="${2:-$RUN_DIR}"
+    mkdir -p "$ARCHIVE_DIR"
+    dest="$ARCHIVE_DIR/$(basename "$run")"
+    mv "$run" "$dest"
+    echo "archived to $dest"
+    ;;
+  serve|render|post) echo "fake magpie: $1 reached the CLI on a local run" ;;
+  *) echo "fake magpie: unsupported subcommand: $*" >&2; exit 64 ;;
+esac
+SHIM
+chmod +x "$HOME/shims/magpie"
+
+cat > "$HOME/shims/gh" <<'SHIM'
+#!/usr/bin/env bash
+. "$HOME/shim-config"
+echo "gh $*" >> "$CALLS"
+echo "fake gh: no request was made" >&2
+exit 1
+SHIM
+chmod +x "$HOME/shims/gh"
+
+# The repo under review: main holds the cache without a reader, the branch adds
+# put/get and a cached load. The run's own files stay out of the diff.
+export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null
+git init -q -b main .
+git config commit.gpgsign false
+printf 'runs/\narchive/\n.magpie-calls.log\n' >> .git/info/exclude
+export GIT_AUTHOR_NAME="Asha Platform" GIT_AUTHOR_EMAIL="asha@example.com"
+export GIT_COMMITTER_NAME="Asha Platform" GIT_COMMITTER_EMAIL="asha@example.com"
+mkdir -p src/settings
+
+cat > src/settings/cache.ts <<'TS'
+const store = new Map<string, Settings>()
+
+export function clear() {
+  store.clear()
+}
+TS
+
+cat > src/settings/loader.ts <<'TS'
+type Settings = { theme: string }
+
+async function fetchSettings(tenantId: string): Promise<Settings> {
+  return { theme: 'default' }
+}
+
+export async function load(tenantId: string) {
+  return fetchSettings(tenantId)
+}
+TS
+
+git add src
+git commit -q -m "feat: tenant settings loader"
+git update-ref refs/remotes/origin/main HEAD
+git symbolic-ref refs/remotes/origin/HEAD refs/remotes/origin/main
+MERGE_BASE=$(git rev-parse HEAD)
+
+git checkout -q -b "$BRANCH"
+
+cat > src/settings/cache.ts <<'TS'
+const store = new Map<string, Settings>()
+
+export function put(tenantId: string, settings: Settings) {
+  store.set(tenantId, settings)
+}
+
+export function get(tenantId: string): Settings | undefined {
+  return store.get(tenantId)
+}
+
+export function clear() {
+  store.clear()
+}
+TS
+
+cat > src/settings/loader.ts <<'TS'
+import { get, put } from './cache'
+
+type Settings = { theme: string }
+
+async function fetchSettings(tenantId: string): Promise<Settings> {
+  return { theme: 'default' }
+}
+
+export async function load(tenantId: string) {
+  const hit = get(tenantId)
+  if (hit) return hit
+  const fresh = await fetchSettings(tenantId)
+  put(tenantId, fresh)
+  return fresh
+}
+TS
+
+git add src
+git commit -q -m "feat: cache tenant settings in the request path" \
+  -m "Settings are read on every request; keep them in memory per tenant."
+HEAD_OID=$(git rev-parse HEAD)
+git diff --no-color --no-ext-diff "$MERGE_BASE" > "$RUN_DIR/diff.patch"
+DIFF_SHA=$(shasum -a 256 "$RUN_DIR/diff.patch" | cut -d' ' -f1)
+
+cat > "$RUN_DIR/target.json" <<JSON
+{ "mode": "local", "root": "$PWD" }
+JSON
+
+cat > "$RUN_DIR/local.json" <<JSON
+{
+  "branch": "$BRANCH",
+  "baseRef": "origin/HEAD",
+  "mergeBase": "$MERGE_BASE",
+  "head": "$HEAD_OID",
+  "dirty": false,
+  "author": "Asha Platform",
+  "commits": [
+    {
+      "oid": "$HEAD_OID",
+      "headline": "feat: cache tenant settings in the request path",
+      "body": "Settings are read on every request; keep them in memory per tenant."
+    }
+  ],
+  "intentFile": null
+}
+JSON
+
+cat > "$RUN_DIR/diff-source.json" <<JSON
+{
+  "source": "local",
+  "mergeBase": "$MERGE_BASE",
+  "baseRef": "origin/HEAD",
+  "head": "$HEAD_OID",
+  "diffSha256": "$DIFF_SHA"
+}
+JSON
+
+cat > "$RUN_DIR/log.jsonl" <<'LOG'
+{"stage":"preflight","status":"done","missingOptional":["codex"]}
+{"stage":"setup","status":"done"}
+{"stage":"context","status":"done"}
+{"stage":"specialists","status":"done"}
+{"stage":"dedupe","status":"done"}
+{"stage":"critic","status":"running"}
+{"stage":"critic","status":"done","kept":2,"dropped":0,"merged":0,"capped":0}
+{"stage":"peer-review","status":"done","provider":"claude"}
+LOG
+
+# Each finding cites a line the branch added, with the evidence that line holds.
+cat > "$RUN_DIR/findings.final.json" <<'JSON'
+[
+  {
+    "id": "security-1",
+    "file": "src/settings/cache.ts",
+    "line": 4,
+    "evidence": "store.set(tenantId, settings)",
+    "severity": "high",
+    "risk": { "impact": "high", "likelihood": "likely", "confidence": "high", "action": "must-fix" },
+    "domain": "security",
+    "title": "Tenant settings cache is a process-global Map with no eviction",
+    "description": "Observation: put() writes into a module-level Map with no bound and no TTL.\n\nWhy it matters: a settings change never reaches the cached copy.\n\nSuggested direction: bound the map and give entries a TTL."
+  },
+  {
+    "id": "bugs-1",
+    "file": "src/settings/loader.ts",
+    "line": 12,
+    "evidence": "const fresh = await fetchSettings(tenantId)",
+    "severity": "medium",
+    "risk": { "impact": "medium", "likelihood": "possible", "confidence": "medium", "action": "should-fix" },
+    "domain": "bugs",
+    "title": "Concurrent loads for the same tenant each hit the network",
+    "description": "Observation: load() awaits fetchSettings before writing back.\n\nWhy it matters: N concurrent first requests produce N fetches.\n\nSuggested direction: cache the in-flight promise."
+  }
+]
+JSON
+
+echo '[]' > "$RUN_DIR/findings/tests.json"

package/skills/magpie/evals/local-mode-hands-back/graders/branch-was-reachable.md

@@ -0,0 +1,5 @@
+---
+type: regex
+pattern: 'export function put\(tenantId'
+target: { source: file, path: src/settings/cache.ts }
+---

package/skills/magpie/evals/local-mode-hands-back/graders/cleanup-after-handoff.md

@@ -0,0 +1,6 @@
+---
+type: regex
+pattern: 'magpie handoff [\s\S]*magpie cleanup \S+'
+target: { source: file, path: .magpie-calls.log }
+weight: 2
+---

package/skills/magpie/evals/local-mode-hands-back/graders/handoff-was-run.md

@@ -0,0 +1,6 @@
+---
+type: regex
+pattern: 'magpie handoff \S+'
+target: { source: file, path: .magpie-calls.log }
+weight: 2
+---

package/skills/magpie/evals/local-mode-hands-back/graders/nothing-served-or-posted.md

@@ -0,0 +1,7 @@
+---
+type: regex
+pattern: 'magpie (serve|render|post)|(^|\n)gh '
+match: not_contains
+target: { source: file, path: .magpie-calls.log }
+weight: 3
+---

package/skills/magpie/evals/local-mode-hands-back/graders/reply-lists-both-findings.md

@@ -0,0 +1,10 @@
+---
+type: llm
+focus: last_message
+weight: 2
+---
+
+This is a local review: the caller is an agent working on its own branch, and there is no PR, server or report page. The run has just printed its handoff and archived itself.
+
+PASS if the reply gives the caller both findings by title, "Tenant settings cache is a process-global Map with no eviction" and "Concurrent loads for the same tenant each hit the network" (light rewording is fine), followed by a line of the form `Run archived at <path>`, and ends the turn there.
+FAIL if either finding is missing, if the `Run archived at <path>` line is missing, if the reply offers to post, points at a report URL or page, asks the caller to tick or select findings, or starts fixing the code itself.

package/skills/magpie/evals/local-mode-hands-back/graders/skill-fired.md

@@ -0,0 +1,5 @@
+---
+type: tool_used
+tool: Skill
+input_match: '"skill"\s*:\s*"(?:[\w-]+:)?magpie"'
+---

package/skills/magpie/evals/local-mode-hands-back/prompt.md

@@ -0,0 +1,11 @@
+---
+name: local-mode-hands-back
+description: A local review of the current branch, resumed after peer review, hands the findings back as text, archives the run, and stops. There is no PR, so nothing is served, rendered or posted.
+tags: [magpie, local, handoff, resume, scaffold]
+max_turns: 25
+timeout_seconds: 900
+allowed_tools: [Read, Glob, Grep, Skill, Write, Edit, Bash, Task]
+expected_outcome: Finds the active local run, runs magpie handoff, then magpie cleanup, and replies with both findings and the archive path. Never serves, renders, posts or calls gh.
+---
+
+Peer review's finished on the magpie local review of this branch. Take it from there.

package/skills/magpie/evals/quality/README.md

@@ -118,9 +118,12 @@ the worktree, so cost grows with candidate count. The results file records the
 ## Full replay
 
 ```
-bun evals/quality/replay-full.ts --corpus <runId> --repo <path-to-local-clone> [--max-cost-usd N]
+bun evals/quality/replay-full.ts --corpus <runId> --repo <path-to-local-clone> [--max-cost-usd N] [--judge-model M]
 ```
 
+Pass `--corpus` as a bare run id: a trailing slash empties the run id, and recall is then
+skipped silently.
+
 Re-runs stages 3 to 6 (context, specialists, dedupe, critic) on the run's `diff.patch`:
 
 1. Scaffolds a scratch run dir: worktree at the PR head as above, `pr.json` and
@@ -132,10 +135,78 @@ Re-runs stages 3 to 6 (context, specialists, dedupe, critic) on the run's `diff.
    is written. Serve, peer review, report, post and cleanup are skipped.
 3. Matches the new kept findings to `findings.final.json` with `matchFindings` and scores
    them; unmatched findings count as `unlabelled` and stay out of precision. Writes
-   `results/<ts>-full-<runId>.json`.
+   `results/<ts>-full-<runId>.json` with `recall`, `recallVerdicts`, `judgeModel` and
+   `judgeCostUsd` all null, plus `log`, `shardManifest`, `findingsFiles`, `dedupedCount`,
+   `criticDropped`, `evidenceDropped` and `thresholdDropped` read from the scratch run dir
+   before it is deleted. `shardCoverage` is derived from the manifest and the findings file
+   names: a shard counts as reviewed when any specialist wrote a file for it, so it overstates
+   a shard only some focuses reviewed. The printed line ends with `, shards <reviewed>/<total>`.
+4. When the run has a recall set (see [Recall](#recall)), judges the new kept findings
+   against it, scores them, and rewrites the same result file with those four fields
+   filled. The printed line gains `, missed-recall <caught>/<misses>`. Without a recall
+   set the fields stay null and nothing extra is printed. A judge failure exits 1 and
+   names the result file, which already holds the scored replay.
 
 **Cost:** a whole review: a scout, five specialists per shard, and the critic batches.
 `--max-cost-usd` (default 10) is passed to Claude as `--max-budget-usd`, the CLI's own
 spend cap; `claude -p` has no `--max-cost-usd` flag. Hitting the cap stops the session,
 and the replay then fails without scoring, on Claude's error result or on the missing
 `findings.kept.json`.
+
+## Recall
+
+The labels above only score findings magpie raised. The recall set scores the other side:
+defects a human found in a reviewed PR that no kept finding raised. It lives at
+`~/.magpie/corpus/misses/<runId>.json`, outside the repo for the same reason as the
+corpus, and is written by hand; corpus rebuilds do not touch it.
+
+Each file is a JSON array of misses:
+
+```json
+{
+  "id": "miss-1",
+  "file": "src/handler.ts",
+  "line": 42,
+  "title": "Retry loop never gives up",
+  "summary": "What the defect is and why it matters.",
+  "ownerDomain": "architecture",
+  "bugClass": "spec/intent mismatch",
+  "outsideDiff": false,
+  "preExisting": false,
+  "source": "https://github.com/<owner>/<repo>/pull/50#discussion_r123"
+}
+```
+
+`ownerDomain` is the specialist domain that should have caught it, `outsideDiff` marks a
+defect in code the diff did not touch, `preExisting` one that predates the PR, and
+`source` is the reviewer comment URL or the fixing commit. `line` may be null. Every
+other field is required; a missing or mistyped one fails `loadMisses`, naming the file
+and index.
+
+The set was seeded from the 2026-10-17 miss analysis, which collected defects from three
+sources per run: reviewer comments magpie had not raised, commits pushed after the review
+that fixed something, and follow-up fix PRs.
+
+`replay-full.ts` hands the misses and the replay's kept findings to a judge: one
+`claude -p` with tools and MCP servers turned off, which names, for each miss, the kept
+finding that describes the same defect wherever it is anchored, or null. User settings
+still load, so `~/.claude/CLAUDE.md` and hooks reach the judge as they would any session.
+`scoreRecall` turns the verdicts into `caught` over `misses`, broken down by owner
+domain and bug class, plus the same figure excluding pre-existing defects. The judge model
+is pinned to `claude-sonnet-5-5` (`DEFAULT_JUDGE_MODEL` in `recall.ts`) so scores stay
+comparable across replays; `--judge-model` overrides it.
+
+**Cost:** one tool-less call per replay, small next to the replay itself. The result file
+records it as `judgeCostUsd`.
+
+**Confounds:**
+
+- **Small set.** 30 misses across 6 runs, so one verdict moves a run's figure a lot.
+- **PR 50 dominates.** Its run holds 14 of the 30, so pooled recall is mostly PR 50's.
+- **Judge nondeterminism.** The same findings can draw different verdicts on another call,
+  most often on partial overlaps. Treat a change of one or two caught misses as noise.
+- **Replay coverage.** Only `pr-50-1789099641` and `pr-70-1790391863` can be replayed
+  here, which covers 19 of the 30 misses. `pr-46` and `pr-38` have no corpus run, since
+  nothing was posted on them and so they have no labels; their misses can never be scored
+  by a replay (and `pr-38` has no local clone either). `pr-231` and `pr-232` are corpus
+  runs, but their repo (INX-Software/WorkforceManagement) is not cloned here.