@jagreehal/shepherd 0.13.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.ts CHANGED
@@ -25,6 +25,13 @@
25
25
  // shepherd acceptance <suggest|local> [--pr <n>] -- <agent command>... (run it; check it kept to the mode)
26
26
  // shepherd trace event <stage> <start|end> --pr <n> [--round <n>] [--outcome ok|failed|skipped] [--retries <n>] [--head <sha>]
27
27
  // shepherd stack <pr> (JSON: a stacked parent and the range to review, or a wrong base)
28
+ // shepherd interdiff <since> <base> [--head <sha>] [--since-base <sha>] (JSON: how the PR's own change moved since a reviewed head)
29
+ // shepherd coverage <round.json> --diff <path> (JSON: changed ranges no finished lens reached; non-zero when any)
30
+ // shepherd reviewed <summary.md> (JSON: the head and base tip the previous round reviewed)
31
+ // shepherd eval export <case-id> <dir> | eval import <case-id> <comments.json> --config <label> --login <reviewer> [--cost <usd>]
32
+ // shepherd impact <diff> (JSON: changed exports, files naming them, each source's tests, stale references)
33
+ // shepherd number <diff> (the diff with each line's old and new numbers, under a contents list)
34
+ // shepherd ci-log <log> (JSON: a failed job log's error lines, and a window on the last error)
28
35
  // shepherd eval run <cases-dir> --config <label> [--repeat <n>] [--out <dir>] -- <agent command>... (review each case, keep each run)
29
36
  // shepherd eval score <runs-dir> [--cases <dir>] [--json] (precision and recall per configuration, dev and held-out apart)
30
37
  // shepherd workflows [--audit] [--upgrade] [--model <id>] [--environment <name>] [--label <name>] [--aws-region <region>] (copy the shepherd and incident workflows, and the audit one, into .github/workflows)
@@ -38,23 +45,26 @@
38
45
  // personal preview that prints locally and never posts. Model ids pass straight to the harness's agent
39
46
  // tool (`haiku` on Claude Code, `provider/model` on OpenCode); --model on a lens pins it to one.
40
47
  import { execFileSync, spawnSync } from "node:child_process";
41
- import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
48
+ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
42
49
  import { homedir, tmpdir } from "node:os";
43
50
  import path from "node:path";
44
51
  import { parseArgs } from "node:util";
45
52
  import { parse as parseYaml, stringify } from "yaml";
46
53
  import { z } from "zod";
47
54
  import { branding, CI_DEFAULT_MODEL, ciModel, ciSettings, type Config, ConfigSchema, defaultBranchDir, defaultBranchReader, type Effective, fetchPackLenses, ghFetch, ghFetchDir, loadPolicy, policyProblems, PRESETS, preview, type Read, TEAM_FILES, configJsonSchema, schemaLine } from "./config.ts";
48
- import { renderReview, renderSummary } from "./summary.ts";
55
+ import { commentableLines, renderReview, renderSummary, reviewedTips, roundCoverage } from "./summary.ts";
49
56
  import { ghThreads, snapshot, verdict } from "./acceptance.ts";
50
57
  import { doctor } from "./doctor.ts";
51
58
  import { fileLedger, notify, notifyOn, STATES, statusLedger } from "./notify.ts";
52
59
  import { collect, collectEscapes, score } from "./garden.ts";
53
60
  import { collectTraces, tempoHeaders, traceLimits, traceScores } from "./garden-traces.ts";
54
61
  import { environmentWorkflow, install, type Outcome, pinWorkflow, uninstall, upgradeWorkflow } from "./install.ts";
55
- import { loadCases, loadRuns, pick, prepare, runCase, scoreRuns, treeHash } from "./eval.ts";
56
- import { buildDemo, captureChange, demoBugs, finish, receiptStatus } from "./local.ts";
57
- import { prStack } from "./stack.ts";
62
+ import { exportCase, importRun, loadCases, PostedComments, loadRuns, pick, prepare, runCase, scoreRuns, treeHash } from "./eval.ts";
63
+ import { buildDemo, captureChange, changedRanges, demoBugs, finish, receiptStatus } from "./local.ts";
64
+ import { analyzeLog } from "./cilog.ts";
65
+ import { numberDiff } from "./diff.ts";
66
+ import { formatLeads, leadsAt } from "./leads.ts";
67
+ import { prInterdiff, prStack } from "./stack.ts";
58
68
  import { flush, recordEvent, traceFile } from "./trace.ts";
59
69
  import { addLens, BUILT_IN_LENSES, ciOpencodeConfig, PROVIDER_KEY_ENVS, pluginRoots, lensTemplates, globalLensFile, lensProblems, newLens, opencodeAgents, opencodeModel, providerKeyEnv, setTier, detectStacks, stackLens, pinModel, repoLensFile, resolveSkill, setCi, setLadder, skillDescription, skillDirs, type Lens } from "./lens.ts";
60
70
 
@@ -91,8 +101,16 @@ const USAGE = `usage: shepherd install [--project | --target <dir>] [--link] [
91
101
  shepherd trace event <stage> <start|end> --pr <n> [--round <n>] [--outcome ok|failed|skipped] [--retries <n>] [--head <sha>]
92
102
  shepherd trace flush --pr <n> --repo <owner/repo> --head <sha> --outcome <how it ended> [--base <sha>] [--ci-run <id> --ci-attempt <n>] [--related <traceparent>]...
93
103
  shepherd stack <pr>
104
+ shepherd interdiff <since> <base> [--head <sha>] [--since-base <sha>]
105
+ shepherd impact <diff>
106
+ shepherd number <diff>
107
+ shepherd ci-log <log>
108
+ shepherd reviewed <summary.md>
109
+ shepherd coverage <round.json> --diff <path>
94
110
  shepherd eval run <cases-dir> --config <label> [--repeat <n>] [--out <dir>] -- <agent command>...
95
111
  shepherd eval score <runs-dir> [--cases <dir>] [--json]
112
+ shepherd eval export <case-id> <dir>
113
+ shepherd eval import <case-id> <comments.json> --config <label> --login <reviewer> [--cost <usd>] [--duration-ms <n>]
96
114
  shepherd model-id <provider:model | provider/model>
97
115
  shepherd workflows [--audit] [--upgrade] [--model <id>] [--environment <name>] [--label <name>] [--aws-region <region>]
98
116
  shepherd notify --pr <n> --repo <owner/repo> --head <sha> --state success|failure|pending|error --file <description.txt>`;
@@ -137,7 +155,12 @@ const { values: opts, positionals } = parseArgs({
137
155
  round: { type: "string" }, // trace: the quality-loop round
138
156
  outcome: { type: "string" }, // trace: how a step or iteration ended
139
157
  retries: { type: "string" },
140
- head: { type: "string" }, // trace, notify: the head SHA
158
+ head: { type: "string" }, // trace, notify, interdiff: the head SHA
159
+ diff: { type: "string" }, // inline, coverage: the reviewed diff
160
+ "since-base": { type: "string" }, // interdiff: the base tip the earlier head was reviewed against
161
+ cost: { type: "string" }, // eval import: what the outside reviewer's run cost, in USD
162
+ login: { type: "string" }, // eval import: the reviewer's GitHub login, whose top-level comments are the findings
163
+ "duration-ms": { type: "string" }, // eval import: how long it took
141
164
  state: { type: "string" }, // notify: the commit status state just set
142
165
  repo: { type: "string" }, // trace flush: owner/repo
143
166
  "ci-run": { type: "string" },
@@ -395,6 +418,30 @@ if (command === "install") {
395
418
  } else if (command === "stack" && sub) {
396
419
  // Which diff to review: the delta over an open stacked parent, or nothing at all when the base is wrong.
397
420
  console.log(JSON.stringify(orExit(() => prStack(Number(sub), repoRoot)), null, 2));
421
+ } else if (command === "coverage" && sub && opts.diff) {
422
+ // Before a round posts: did every changed range of the PR's diff reach a lens that finished? Non-zero when not.
423
+ const result = orExit(() => roundCoverage(JSON.parse(readFileSync(sub, "utf8")), readFileSync(opts.diff!, "utf8")));
424
+
425
+ console.log(JSON.stringify(result, null, 2));
426
+
427
+ if (!result.complete) process.exitCode = 1;
428
+ } else if (command === "reviewed" && sub) {
429
+ // The head and base tip the previous round reviewed, from its summary comment: the next round's since_sha and --since-base.
430
+ console.log(JSON.stringify(reviewedTips(readFileSync(sub, "utf8"))));
431
+ } else if (command === "impact" && sub) {
432
+ // Where the change may reach beyond its diff: changed exports, the files naming them, each changed source's tests.
433
+ const patch = readFileSync(sub, "utf8");
434
+ const changed = [...new Set(changedRanges(patch).map((r) => r.file))];
435
+ const leads = leadsAt(repoRoot, patch, changed) ?? { symbols: [], sources: [], references: [] };
436
+
437
+ console.log(JSON.stringify({ ...leads, lines: formatLeads(leads, (n) => n.replace(/[\n\r\t]+/g, " ")) }, null, 2));
438
+ } else if (command === "number" && sub) {
439
+ console.log(numberDiff(readFileSync(sub, "utf8")));
440
+ } else if (command === "ci-log" && sub) {
441
+ console.log(JSON.stringify(analyzeLog(readFileSync(sub, "utf8")), null, 2));
442
+ } else if (command === "interdiff" && sub && skillRef) {
443
+ // A later round's focus: the PR's own change since the head swarm last reviewed, without the base's commits.
444
+ console.log(JSON.stringify(orExit(() => prInterdiff(sub, opts.head ?? "HEAD", skillRef, repoRoot, opts["since-base"])), null, 2));
398
445
  } else if (command === "workflows") {
399
446
  const dest = path.join(repoRoot, ".github", "workflows");
400
447
 
@@ -598,7 +645,11 @@ if (command === "install") {
598
645
  // The checkout is the reviewed head: suggestions are placed against its files, read from the working directory.
599
646
  const read = (p: string) => (existsSync(p) ? readFileSync(p, "utf8") : null);
600
647
 
601
- console.log(JSON.stringify(orExit(() => renderReview(input, b.emoji, b.headers.swarm, read, reviewThreads())), null, 2));
648
+ // With --diff, a finding off the diff's hunks stays out of the review, which GitHub would reject whole.
649
+ const diff = opts.diff ? readFileSync(opts.diff, "utf8") : undefined;
650
+ const [commentable, removable] = diff === undefined ? [] : [commentableLines(diff), commentableLines(diff, "LEFT")];
651
+
652
+ console.log(JSON.stringify(orExit(() => renderReview(input, b.emoji, b.headers.swarm, read, reviewThreads(), commentable, removable)), null, 2));
602
653
  } else if (command === "fanout") {
603
654
  // Swarm's first wave, dispatched in one message beside the router: correctness on the whole diff and every custom
604
655
  // lens whose applies_to matches a changed file, with its directory. The CLI lists them so no model skips one.
@@ -673,6 +724,29 @@ if (command === "install") {
673
724
  console.log(`${run.status.padEnd(10)} ${c.id} #${n} ${run.findings.length} finding(s) ${(run.duration_ms / 1000).toFixed(0)}s${run.cost_usd === null ? "" : ` $${run.cost_usd.toFixed(2)}`}${run.reason ? ` ${run.reason}` : ""}`);
674
725
  }
675
726
  }
727
+ } else if (command === "eval" && sub === "export" && skillRef && rest[0]) {
728
+ // One case as a repository another reviewer can review as a PR: main at its base, pr at its head.
729
+ const casesDir = path.resolve(opts.cases ?? path.join(repoRoot, "evals", "cases"));
730
+ const c = orExit(() => loadCases(casesDir).find((x) => x.id === skillRef) ?? (() => { throw new Error(`no case ${skillRef} under ${casesDir}`); })());
731
+ const caseRoot = execFileSync("git", ["-C", casesDir, "rev-parse", "--show-toplevel"], { encoding: "utf8" }).trim();
732
+ const done = orExit(() => exportCase(c, caseRoot, path.resolve(rest[0]!)));
733
+
734
+ console.log(`${done.tree}: main at ${done.base.slice(0, 7)}, pr at ${done.head.slice(0, 7)}. Push both, open pr against main, then: shepherd eval import ${c.id} <comments.json> --config <label> --login <reviewer-login>`);
735
+ } else if (command === "eval" && sub === "import" && skillRef && rest[0] && opts.config && opts.login) {
736
+ // Another reviewer's inline comments on an exported case PR, scored by eval score beside shepherd's runs.
737
+ const cases = orExit(() => loadCases(path.resolve(opts.cases ?? path.join(repoRoot, "evals", "cases"))));
738
+ const c = cases.find((x) => x.id === skillRef) ?? orExit(() => { throw new Error(`no case ${skillRef}`); });
739
+ const out = path.resolve(opts.out ?? path.join(repoRoot, "evals", "runs"), opts.config.replace(/[^\w.-]/g, "_"), "imported");
740
+ // Each import is the next run of this case: one more review, never a replacement for an earlier one.
741
+ const n = existsSync(out) ? readdirSync(out).filter((f) => f.startsWith(`${c.id}-`) && /^\d+\.json$/.test(f.slice(c.id.length + 1))).length : 0;
742
+
743
+ const run = orExit(() =>
744
+ importRun(c, PostedComments.parse(JSON.parse(readFileSync(rest[0]!, "utf8"))), { config: opts.config!, run: n, login: opts.login!, cost_usd: opts.cost === undefined ? null : Number(opts.cost), duration_ms: Number(opts["duration-ms"] ?? 0) }),
745
+ );
746
+
747
+ mkdirSync(out, { recursive: true });
748
+ writeFileSync(path.join(out, `${c.id}-${n}.json`), `${JSON.stringify(run, null, 2)}\n`);
749
+ console.log(`imported ${run.findings.length} finding(s) by ${opts.login} as ${path.join(out, `${c.id}-${n}.json`)}`);
676
750
  } else if (command === "eval" && sub === "score" && skillRef) {
677
751
  const cases = orExit(() => loadCases(path.resolve(opts.cases ?? path.join(repoRoot, "evals", "cases"))));
678
752
  const rows = orExit(() => scoreRuns(cases, loadRuns(path.resolve(skillRef))));
package/src/diff.ts ADDED
@@ -0,0 +1,70 @@
1
+ // What a reviewer reads: a diff with its line numbers written out and a contents list.
2
+ import { unquotePath } from "./local.ts";
3
+
4
+ /**
5
+ * Files whose hunks carry nothing a reviewer weighs: lockfiles, minified and generated output, snapshots, images and
6
+ * fonts, built trees, migration metadata. `build/` and `vendor/` stay out: source lives under both.
7
+ */
8
+ export const GENERATED =
9
+ /(^|\/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|Cargo\.lock|go\.sum|poetry\.lock|Pipfile\.lock|Gemfile\.lock|composer\.lock|.*\.snap|.*\.min\.(js|css)|.*\.map|.*\.lock|.*\.(pb\.go|_pb2\.py|g\.dart)|.*\.(generated|gen)\.[^/]+|.*\.(svg|png|jpe?g|gif|ico|webp|woff2?|ttf|pdf))$|(^|\/)(dist|generated|__generated__|node_modules|\.yarn|__snapshots__|coverage|\.next|\.turbo|drizzle\/meta)\//i;
10
+
11
+ const pad = (n: number) => String(n).padStart(4, " ");
12
+
13
+ /**
14
+ * A unified diff with each hunk line's old and new numbers beside it, `| OLD | NEW | TYPE | code`, under a contents
15
+ * list giving each file's lines in this output, so a reviewer cites a line without counting from a hunk header.
16
+ * Generated files are flagged in the contents and kept in the body. `coverage` and `inline` read the raw diff.
17
+ */
18
+ export function numberDiff(patch: string): string {
19
+ const files: { name: string; lines: string[] }[] = [];
20
+ let file: (typeof files)[number] | undefined;
21
+ let [oldAt, newAt, oldLeft, newLeft] = [0, 0, 0, 0];
22
+
23
+ for (const l of patch.split("\n")) {
24
+ if (oldLeft > 0 || newLeft > 0) {
25
+ if (l.startsWith("\\")) file?.lines.push(l);
26
+ else if (l.startsWith("+")) {
27
+ file?.lines.push(`| | ${pad(newAt++)} | + | ${l.slice(1)}`);
28
+ newLeft--;
29
+ } else if (l.startsWith("-")) {
30
+ file?.lines.push(`| ${pad(oldAt++)} | | - | ${l.slice(1)}`);
31
+ oldLeft--;
32
+ } else {
33
+ file?.lines.push(`| ${pad(oldAt++)} | ${pad(newAt++)} | | ${l.slice(1)}`);
34
+ oldLeft--;
35
+ newLeft--;
36
+ }
37
+
38
+ continue;
39
+ }
40
+
41
+ const hunk = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/.exec(l);
42
+
43
+ if (l.startsWith("diff --git ")) {
44
+ file = { name: / b\/(.+)$/.exec(l)?.[1] ?? l.slice(11), lines: [] };
45
+ files.push(file);
46
+ } else if (hunk) {
47
+ [oldAt, oldLeft, newAt, newLeft] = [Number(hunk[1]), Number(hunk[2] ?? 1), Number(hunk[3]), Number(hunk[4] ?? 1)];
48
+ } else if (file && (l.startsWith("+++ ") || l.startsWith("--- "))) {
49
+ // `+++` names the file as it is now; a deleted file's only name is on `---`.
50
+ const v = unquotePath(l.slice(4).replace(/\t$/, ""));
51
+
52
+ if (v !== "/dev/null") file.name = v.replace(/^[ab]\//, "");
53
+ }
54
+
55
+ if (l || file?.lines.length) file?.lines.push(l);
56
+ }
57
+
58
+ // The contents take one line per file plus a heading, a blank line, `---` and a blank line.
59
+ let at = files.length + 5;
60
+
61
+ const contents = files.map((f) => {
62
+ const start = at;
63
+
64
+ at += f.lines.length;
65
+
66
+ return `- ${f.name} → lines ${start}-${at - 1}${GENERATED.test(f.name) ? " · generated" : ""}`;
67
+ });
68
+
69
+ return [`## Files (${files.length})`, ...contents, "", "---", "", ...files.flatMap((f) => f.lines)].join("\n");
70
+ }
package/src/eval.ts CHANGED
@@ -7,7 +7,7 @@
7
7
  */
8
8
  import { spawnSync } from "node:child_process";
9
9
  import { createHash } from "node:crypto";
10
- import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
10
+ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
11
11
  import path from "node:path";
12
12
  import { z } from "zod";
13
13
  import { buildDemo, demoBugs, finish, git, ReviewRecord, stateDir } from "./local.ts";
@@ -319,3 +319,59 @@ export function runCase(c: EvalCase, prepared: Prepared, agent: string[], meta:
319
319
 
320
320
  return EvalRun.parse({ ...base, status: checked.status, reason: [...checked.errors, ...checked.missing].join("; "), findings: record.success ? record.data.findings : [] });
321
321
  }
322
+
323
+ /**
324
+ * The case as a repository another reviewer can open as a PR: a clone with `main` at the case's base and `pr` at its
325
+ * head, checked out on `pr`. Push both to a scratch repository, open `pr` against `main`, and import what it posts.
326
+ */
327
+ export type Exported = { tree: string; base: string; head: string };
328
+
329
+ export function exportCase(c: EvalCase, caseRoot: string, dir: string): Exported {
330
+ if ("demo" in c.source) throw new Error(`${c.id} is the demo case, which builds itself: run it with eval run`);
331
+ mkdirSync(dir, { recursive: true });
332
+ const { tree, base } = prepare(c, caseRoot, dir);
333
+ const head = c.source.head;
334
+
335
+ git(tree, ["branch", "-f", "main", base]);
336
+ git(tree, ["checkout", "-q", "-B", "pr", head]);
337
+
338
+ return { tree, base, head };
339
+ }
340
+
341
+ // Inline comments as `gh api repos/<o>/<r>/pulls/<n>/comments --paginate` lists them: who wrote each, and the
342
+ // comment it replies to, so replies and other people's comments stay out of the run.
343
+ export const PostedComments = z.array(
344
+ z.object({
345
+ path: z.string(),
346
+ line: z.number().int().positive().nullable().optional(),
347
+ original_line: z.number().int().positive().nullable().optional(),
348
+ body: z.string().min(1),
349
+ user: z.object({ login: z.string() }),
350
+ in_reply_to_id: z.number().int().nullable().optional(),
351
+ }),
352
+ );
353
+
354
+ export type PostedComments = z.infer<typeof PostedComments>;
355
+
356
+ /**
357
+ * Another reviewer's inline comments on an exported case PR, as a run `eval score` reads like any other: only the
358
+ * comments `login` wrote that open a thread. A reply ("Thanks, fixed") is no finding, and neither is anyone else's
359
+ * comment. Every finding counts as actionable (MEDIUM): an outside reviewer marks no NITs shepherd can read, so its
360
+ * precision counts them all.
361
+ */
362
+ export function importRun(c: EvalCase, comments: PostedComments, meta: { config: string; run: number; login: string; cost_usd: number | null; duration_ms: number }): EvalRun {
363
+ const posted = comments.filter((p) => p.user.login === meta.login && p.in_reply_to_id == null);
364
+
365
+ return EvalRun.parse({
366
+ case: c.id,
367
+ config: meta.config,
368
+ run: meta.run,
369
+ shepherd: `external:${meta.login}`,
370
+ agent: [meta.login],
371
+ started_at: new Date().toISOString(),
372
+ duration_ms: meta.duration_ms,
373
+ cost_usd: meta.cost_usd,
374
+ status: "complete",
375
+ findings: posted.map((p) => ({ file: p.path, line: p.line ?? p.original_line ?? "general", severity: "MEDIUM", lens: meta.login, body: p.body })),
376
+ });
377
+ }
package/src/garden.ts CHANGED
@@ -25,6 +25,8 @@ export type PrRecord = {
25
25
 
26
26
  const DAY = 864e5;
27
27
 
28
+ type Outcome = "fixed" | "dismissed" | "argued" | "open";
29
+
28
30
  export type Signal = "reverted_fix" | "human_replied" | "round_cap" | "stopped_short" | "loop_stalled" | "churn";
29
31
 
30
32
  const TAG = /\*\*\[(?:convergent: )?([^\]]+)\]\*\*\s*\S+\s*(CRITICAL|HIGH|MEDIUM|LOW|NIT)/;
@@ -81,6 +83,14 @@ export function score(all: PrRecord[], minPrs = 3, now = Date.now()) {
81
83
  };
82
84
  });
83
85
 
86
+ const reviewed = prs.flatMap((p) => (p.trusted && p.state === "MERGED" && p.rounds ? [p.rounds] : []));
87
+
88
+ const rounds = {
89
+ prs: reviewed.length,
90
+ mean: reviewed.length ? reviewed.reduce((a, b) => a + b, 0) / reviewed.length : null,
91
+ oneRound: reviewed.length ? reviewed.filter((r) => r === 1).length / reviewed.length : null,
92
+ };
93
+
84
94
  const byRule = new Map<string, { prs: Set<number>; paths: Set<string> }>();
85
95
 
86
96
  // Accepted means triage fixed it and said so in a Shepherd-Fixes trailer. Triage also resolves findings it
@@ -104,7 +114,29 @@ export function score(all: PrRecord[], minPrs = 3, now = Date.now()) {
104
114
  .flatMap(([rule, e]) => (e.prs.size >= minPrs ? [{ rule, prs: [...e.prs].sort((a, b) => a - b), paths: [...e.paths].sort() }] : []))
105
115
  .sort((a, b) => b.prs.length - a.prs.length || a.rule.localeCompare(b.rule));
106
116
 
107
- return { prs: scored, recurring };
117
+ return { prs: scored, recurring, rounds, outcomes: outcomesOf(prs.filter((p) => p.trusted)) };
118
+ }
119
+
120
+ /** Each lens rule's threads by how they ended, noisiest first. Only a rule can match a trailer, so a thread with a bare lens tag is left out. */
121
+ function outcomesOf(prs: PrRecord[]) {
122
+ const byRule = new Map<string, Record<Outcome, number>>();
123
+
124
+ for (const pr of prs) {
125
+ const fixed = new Set(pr.commits.filter((c) => !isReverted(c, pr.commits)).flatMap((c) => c.fixes).flatMap(rulesOf));
126
+
127
+ for (const t of pr.threads) {
128
+ const rules = rulesOf(t.tag);
129
+ const outcome: Outcome = t.humanReplied ? "argued" : !t.resolved ? "open" : rules.some((r) => fixed.has(r)) ? "fixed" : "dismissed";
130
+
131
+ for (const rule of rules) {
132
+ const entry = byRule.get(rule) ?? { fixed: 0, dismissed: 0, argued: 0, open: 0 };
133
+ entry[outcome] += 1;
134
+ byRule.set(rule, entry);
135
+ }
136
+ }
137
+ }
138
+
139
+ return [...byRule].map(([rule, counts]) => ({ rule, ...counts })).sort((a, b) => b.dismissed + b.argued - a.dismissed - a.argued || a.rule.localeCompare(b.rule));
108
140
  }
109
141
 
110
142
  /** Parse one shepherd thread's first comment and its replies; null when the thread is not shepherd's. */
@@ -175,7 +207,7 @@ const Commits = z.array(z.object({ sha: z.string(), commit: z.object({ message:
175
207
 
176
208
  const Statuses = z.object({ statuses: z.array(z.object({ context: z.string(), state: z.string(), description: z.string().nullable(), updated_at: z.string() })) });
177
209
 
178
- const Comments = z.array(z.object({ body: z.string() }));
210
+ const Comments = z.array(z.object({ body: z.string(), user: z.object({ login: z.string(), type: z.string() }).nullable() }));
179
211
 
180
212
  const Issues = z.array(z.object({ number: z.number(), title: z.string(), url: z.string(), body: z.string() }));
181
213
 
@@ -208,16 +240,19 @@ export function collect(repo: string, since: string): PrRecord[] {
208
240
  return canPush.get(login) === true;
209
241
  };
210
242
 
243
+ // Anyone can paste the header on a public PR, so shepherd's marks count only from a bot or an account that can push.
244
+ const canPost = (login: string | undefined, bot: boolean) => bot || (login !== undefined && trusted(login));
245
+
211
246
  return prs.map((pr) => {
212
247
  const query = "query($owner:String!,$name:String!,$n:Int!){repository(owner:$owner,name:$name){pullRequest(number:$n){reviewThreads(first:100){nodes{isResolved path comments(first:50){nodes{author{login __typename} body}}}}}}}";
213
248
 
214
249
  const threads = Threads.parse(JSON.parse(gh(["api", "graphql", "-f", `query=${query}`, "-f", `owner=${owner}`, "-f", `name=${name}`, "-F", `n=${pr.number}`])))
215
- .data.repository.pullRequest.reviewThreads.nodes.flatMap((t) => parseThread(t.path, t.isResolved, t.comments.nodes.map((c) => ({ author: c.author?.login ?? "ghost", bot: c.author?.__typename === "Bot", body: c.body }))) ?? []);
250
+ .data.repository.pullRequest.reviewThreads.nodes.filter((t) => canPost(t.comments.nodes[0]?.author?.login, t.comments.nodes[0]?.author?.__typename === "Bot")).flatMap((t) => parseThread(t.path, t.isResolved, t.comments.nodes.map((c) => ({ author: c.author?.login ?? "ghost", bot: c.author?.__typename === "Bot", body: c.body }))) ?? []);
216
251
 
217
252
  const commits = Commits.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/pulls/${pr.number}/commits`])).flat()).map((c) => parseCommit(c.sha, c.commit.message, c.commit.committer?.date ?? ""));
218
253
 
219
254
  const status = Statuses.parse(JSON.parse(gh(["api", `repos/${repo}/commits/${pr.headRefOid}/status`]))).statuses.find((s) => s.context === "shepherd");
220
- const summary = Comments.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/issues/${pr.number}/comments`])).flat()).find((c) => c.body.includes("<!-- shepherd-swarm-summary -->"));
255
+ const summary = Comments.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/issues/${pr.number}/comments`])).flat()).find((c) => c.body.includes("<!-- shepherd-swarm-summary -->") && canPost(c.user?.login, c.user?.type === "Bot"));
221
256
 
222
257
  return { number: pr.number, headSha: pr.headRefOid, title: pr.title, state: pr.state, threads, commits, status: status ? { state: status.state, description: status.description ?? "", updatedAt: status.updated_at } : null, rounds: roundsOf(summary?.body), trusted: trusted(pr.author.login) };
223
258
  });
package/src/leads.ts ADDED
@@ -0,0 +1,234 @@
1
+ // Impact leads: where a change may reach beyond its diff. Changed exports and the files outside the PR that name
2
+ // them, the test files that sit beside each changed source file, and the lines still naming an identifier the diff
3
+ // renamed or removed. Advisory context only: a lead says where to look, it never stands in for reading the diff, and
4
+ // nothing here gates.
5
+ import { execFileSync } from "node:child_process";
6
+ import { GENERATED } from "./diff.ts";
7
+
8
+ export type Reference = { file: string; line: number; text: string };
9
+
10
+ export type Leads = {
11
+ /** An exported name the diff adds, removes or rewrites, and up to MAX_CALLERS files outside the PR that name it. */
12
+ symbols: { name: string; callers: string[] }[];
13
+ /** Changed source files with the test files that match their name; `tests` is empty when none does. */
14
+ sources: { file: string; tests: string[] }[];
15
+ /**
16
+ * Identifiers whose count the diff changes, and lines it left alone that still name them, docs included: a rename or
17
+ * removal's stale references. `count` is every such line; `refs` the first MAX_REFERENCES.
18
+ */
19
+ references: { name: string; removed: boolean; count: number; refs: Reference[] }[];
20
+ };
21
+
22
+ const MAX_SYMBOLS = 8;
23
+
24
+ const MAX_CALLERS = 5;
25
+
26
+ const MAX_SOURCES = 10;
27
+
28
+ const MAX_CANDIDATES = 24;
29
+
30
+ const MAX_ATOMS = 12;
31
+
32
+ const MAX_REFERENCES = 6;
33
+
34
+ const MAX_EXCERPT = 160;
35
+
36
+ // A declaration's name on a diff line: TS/JS exports, Python top-level defs and classes, Go exported funcs.
37
+ const DECLARATION = [
38
+ /^[+-]\s*export\s+(?:default\s+)?(?:declare\s+)?(?:async\s+)?(?:function\*?|const|let|var|class|interface|type|enum)\s+([A-Za-z_$][\w$]*)/,
39
+ /^[+-](?:def|class)\s+([A-Za-z_]\w*)/,
40
+ /^[+-]func\s+(?:\([^)]*\)\s*)?([A-Z]\w*)/,
41
+ ];
42
+
43
+ const SOURCE = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|java|kt|cs|rb)$/;
44
+
45
+ const TEST_FILE = /(^|\/)(__tests__\/|tests?\/)|[._-](test|spec)\.[^/]+$|(^|\/)test_[^/]+\.py$|_test\.go$/;
46
+
47
+ /** The names the diff's hunk lines declare, in order of first appearance, at most MAX_SYMBOLS. */
48
+ export function changedSymbols(diff: string): string[] {
49
+ const names = new Set<string>();
50
+ let inHunk = false;
51
+
52
+ for (const l of diff.split("\n")) {
53
+ if (l.startsWith("diff --git ")) inHunk = false;
54
+ else if (l.startsWith("@@")) inHunk = true;
55
+ else if (inHunk)
56
+ for (const re of DECLARATION) {
57
+ const name = re.exec(l)?.[1];
58
+
59
+ if (name && name.length > 2) names.add(name);
60
+ }
61
+ }
62
+
63
+ return [...names].slice(0, MAX_SYMBOLS);
64
+ }
65
+
66
+ type AtomChange = { atom: string; confidence: number; added: number; removed: number };
67
+
68
+ /** Identifier-shaped and specific enough to search for: mixed case or an underscore, four characters or more. */
69
+ const searchable = (atom: string) => atom.length >= 4 && atom.length <= 128 && /[A-Z_]/.test(atom);
70
+
71
+ /** How sure the text around a name is that it names a symbol: a declaration, a binding or call, a quoted or dotted use, a bare word. */
72
+ function confidence(text: string, atom: string, at: number): number {
73
+ const before = text.slice(Math.max(0, at - 64), at);
74
+ const after = text.slice(at + atom.length, at + atom.length + 16);
75
+
76
+ if (/\b(?:class|interface|type|enum|function|def|func|fn|struct|trait)\s+$/.test(before)) return 4;
77
+
78
+ if (/\b(?:const|let|var)\s+$/.test(before) || /^\s*\??\s*[:(]/.test(after)) return 3;
79
+
80
+ return (/['"`]$/.test(before) && /^['"`]/.test(after)) || before.endsWith(".") || after.startsWith(".") || atom.includes("_") ? 2 : 1;
81
+ }
82
+
83
+ /**
84
+ * Identifiers whose count the diff changes, surest first, and the added lines to leave out of the search. A name on as
85
+ * many added lines as removed moved or stayed; a rename shows as one name's count falling and another's rising. Names
86
+ * the diff removes rank first.
87
+ */
88
+ export function changedAtoms(diff: string) {
89
+ const changes = new Map<string, AtomChange>();
90
+ const added = new Set<string>();
91
+ let file = "";
92
+ let at = 0;
93
+ let [oldLeft, newLeft] = [0, 0];
94
+
95
+ const record = (text: string, side: "added" | "removed") => {
96
+ for (const m of text.matchAll(/[A-Za-z_][A-Za-z0-9_]*/g)) {
97
+ if (!searchable(m[0])) continue;
98
+ const c = changes.get(m[0]) ?? { atom: m[0], confidence: 0, added: 0, removed: 0 };
99
+
100
+ c.confidence = Math.max(c.confidence, confidence(text, m[0], m.index));
101
+ c[side]++;
102
+ changes.set(m[0], c);
103
+ }
104
+ };
105
+
106
+ for (const l of diff.split("\n")) {
107
+ if (l.startsWith("diff --git ")) [oldLeft, newLeft] = [0, 0];
108
+
109
+ if (oldLeft > 0 || newLeft > 0) {
110
+ if (l.startsWith("+")) {
111
+ added.add(`${file}\0${at}`);
112
+ record(l.slice(1), "added");
113
+ at++;
114
+ newLeft--;
115
+ } else if (l.startsWith("-")) {
116
+ record(l.slice(1), "removed");
117
+ oldLeft--;
118
+ } else if (!l.startsWith("\\")) {
119
+ at++;
120
+ oldLeft--;
121
+ newLeft--;
122
+ }
123
+
124
+ continue;
125
+ }
126
+
127
+ const hunk = /^@@ -\d+(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/.exec(l);
128
+
129
+ if (hunk) [oldLeft, at, newLeft] = [Number(hunk[1] ?? 1), Number(hunk[2]), Number(hunk[3] ?? 1)];
130
+ else if (l.startsWith("+++ b/")) file = l.slice(6).replace(/\t$/, "");
131
+ }
132
+
133
+ const atoms = [...changes.values()]
134
+ .filter((c) => c.added !== c.removed)
135
+ .sort((a, b) => b.confidence - a.confidence || Number(b.removed > 0) - Number(a.removed > 0) || Math.abs(b.added - b.removed) - Math.abs(a.added - a.removed) || b.atom.length - a.atom.length || (a.atom < b.atom ? -1 : 1))
136
+ .slice(0, MAX_CANDIDATES);
137
+
138
+ return { atoms, added };
139
+ }
140
+
141
+ /** A file's language, so a TypeScript source never claims a Go test: JS and TS share one, any other extension is its own. */
142
+ const family = (f: string) => {
143
+ const ext = /\.([^./]+)$/.exec(f)?.[1] ?? "";
144
+
145
+ return /^[cm]?[jt]sx?$/.test(ext) ? "js" : ext;
146
+ };
147
+
148
+ /** Tracked test files whose name matches the source file's stem: `a.ts` → `a.test.ts`, `__tests__/a.ts`, `test_a.py`, `a_test.go`. */
149
+ export function relatedTests(file: string, tracked: string[]): string[] {
150
+ const stem = file.split("/").at(-1)!.replace(/\.[^.]+$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
151
+ const named = new RegExp(`(^|/)(test_)?${stem}([._-](test|spec))?\\.[^/]+$|(^|/)${stem}_test\\.go$`);
152
+
153
+ const lang = family(file);
154
+
155
+ return tracked.filter((t) => t !== file && TEST_FILE.test(t) && named.test(t) && family(t) === lang);
156
+ }
157
+
158
+ export type LeadDeps = {
159
+ /** Tracked files, `git ls-files`. */
160
+ tracked: () => string[];
161
+ /** Tracked files naming `word` as a whole word, `git grep -lw`. */
162
+ naming: (word: string) => string[];
163
+ /** The lines of tracked text files naming `word` as a whole word, `git grep -nw`. */
164
+ lines: (word: string) => Reference[];
165
+ };
166
+
167
+ export function impactLeads(diff: string, changed: string[], deps: LeadDeps): Leads {
168
+ const inPR = new Set(changed);
169
+ const tracked = deps.tracked();
170
+ const { atoms, added } = changedAtoms(diff);
171
+
172
+ const references = atoms.flatMap((a) => {
173
+ const refs = deps.lines(a.atom).filter((r) => !GENERATED.test(r.file) && !added.has(`${r.file}\0${r.line}`));
174
+
175
+ return refs.length ? [{ name: a.atom, removed: a.removed > a.added, count: refs.length, refs: refs.slice(0, MAX_REFERENCES).map((r) => ({ ...r, text: r.text.trim().slice(0, MAX_EXCERPT) })) }] : [];
176
+ });
177
+
178
+ return {
179
+ references: references.slice(0, MAX_ATOMS),
180
+ // A caller is code: a doc or changelog that mentions the name is not one.
181
+ symbols: changedSymbols(diff).map((name) => ({ name, callers: deps.naming(name).filter((f) => !inPR.has(f) && SOURCE.test(f) && !GENERATED.test(f)).slice(0, MAX_CALLERS) })),
182
+ sources: changed
183
+ .filter((f) => !TEST_FILE.test(f) && SOURCE.test(f) && !GENERATED.test(f))
184
+ .slice(0, MAX_SOURCES)
185
+ .map((file) => ({ file, tests: relatedTests(file, tracked).slice(0, 3) })),
186
+ };
187
+ }
188
+
189
+ /** Leads over a checkout, through git: no PR code runs. Null when git fails, so the prompt simply goes without. */
190
+ export function leadsAt(root: string, diff: string, changed: string[]): Leads | null {
191
+ const git = (...args: string[]) => {
192
+ try {
193
+ return execFileSync("git", ["-C", root, "-c", "core.fsmonitor=false", ...args], { encoding: "utf8", maxBuffer: 16 << 20, stdio: ["ignore", "pipe", "ignore"] });
194
+ } catch (e) {
195
+ // git grep exits 1 for "no matches", which is an answer.
196
+ if (e instanceof Error && "status" in e && e.status === 1) return "";
197
+ throw e;
198
+ }
199
+ };
200
+
201
+ // -z: paths as they are, NUL-separated. Without it git quotes a path holding a tab, a quote or non-ASCII bytes
202
+ // (`"src/caf\303\251.ts"`), and the quoted name matches neither the PR's files nor a test's.
203
+ const paths = (s: string) => s.split("\0").filter(Boolean);
204
+
205
+ // -z prints path NUL line NUL text, one match per newline.
206
+ const lines = (w: string) =>
207
+ git("grep", "-nwFIz", "-e", w, "--")
208
+ .split("\n")
209
+ .flatMap((l) => {
210
+ const [file, line, text] = l.split("\0");
211
+
212
+ return file && line && text !== undefined ? [{ file, line: Number(line), text }] : [];
213
+ });
214
+
215
+ try {
216
+ return impactLeads(diff, changed, { tracked: () => paths(git("ls-files", "-z")), naming: (w) => paths(git("grep", "-lwFz", "--", w)), lines });
217
+ } catch {
218
+ return null;
219
+ }
220
+ }
221
+
222
+ /** The leads for the prompt: one line each, names passed through `clean` by the caller. */
223
+ export function formatLeads(leads: Leads, clean: (s: string) => string): string[] {
224
+ const symbols = leads.symbols.map((s) => `${clean(s.name)}: ${s.callers.length ? `named in ${s.callers.map(clean).join(", ")}` : "named nowhere outside this PR"}`);
225
+ const sources = leads.sources.map((s) => `${clean(s.file)}: ${s.tests.length ? `tests in ${s.tests.map(clean).join(", ")}` : "no test file matches its name"}`);
226
+
227
+ const references = leads.references.map((r) => {
228
+ const shown = r.refs.map((x) => `${clean(x.file)}:${x.line} ${JSON.stringify(clean(x.text))}`).join("; ");
229
+
230
+ return `${clean(r.name)}: ${r.removed ? "fewer uses after this diff, still" : "more uses after this diff, also"} named on ${r.count} unchanged line${r.count === 1 ? "" : "s"}: ${shown}`;
231
+ });
232
+
233
+ return [...symbols, ...sources, ...references];
234
+ }