@jagreehal/shepherd 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/AGENTS.md +3 -3
  2. package/README.md +40 -27
  3. package/THIRD_PARTY_NOTICES.md +2 -0
  4. package/gate/.shepherd/gate.schema.json +647 -0
  5. package/gate/.shepherd/gate.yml +122 -0
  6. package/gate/.shepherd/review-guidance.md +95 -0
  7. package/gate/src/actions.ts +32 -0
  8. package/gate/src/cli.ts +1028 -0
  9. package/gate/src/config.ts +102 -0
  10. package/gate/src/delta.ts +29 -0
  11. package/gate/src/env.ts +30 -0
  12. package/gate/src/eval.ts +288 -0
  13. package/gate/src/familiarity.ts +461 -0
  14. package/gate/src/github.ts +761 -0
  15. package/gate/src/inline.ts +59 -0
  16. package/gate/src/isolated-cli.ts +281 -0
  17. package/gate/src/leads.ts +257 -0
  18. package/gate/src/llm.ts +606 -0
  19. package/gate/src/notify.ts +57 -0
  20. package/gate/src/policy.ts +1297 -0
  21. package/gate/src/provider-errors.ts +80 -0
  22. package/gate/src/retention.ts +86 -0
  23. package/gate/src/reviewer.ts +739 -0
  24. package/gate/src/signals.ts +105 -0
  25. package/gate/src/smoke-live.ts +64 -0
  26. package/gate/src/stack.ts +87 -0
  27. package/gate/src/stories.ts +297 -0
  28. package/gate/templates/policy-strict.yml +127 -0
  29. package/gate/templates/reviewer.Dockerfile +8 -0
  30. package/gate/templates/rules/observability/SKILL.md +33 -0
  31. package/gate/templates/shepherd-gate-actions.yml +35 -0
  32. package/gate/templates/shepherd-gate-edited.yml +29 -0
  33. package/gate/templates/shepherd-gate.yml +111 -0
  34. package/lenses/dotnet/SKILL.md +1 -1
  35. package/lenses/go/SKILL.md +1 -1
  36. package/lenses/java/SKILL.md +1 -1
  37. package/lenses/nextjs/SKILL.md +1 -1
  38. package/lenses/node-api/SKILL.md +1 -1
  39. package/lenses/nx-monorepo/SKILL.md +1 -1
  40. package/lenses/observability/SKILL.md +1 -1
  41. package/lenses/owasp-asvs/SKILL.md +1 -1
  42. package/lenses/owasp-llm/SKILL.md +1 -1
  43. package/lenses/pstack/SKILL.md +1 -1
  44. package/lenses/python/SKILL.md +1 -1
  45. package/lenses/react/SKILL.md +1 -1
  46. package/lenses/rust/SKILL.md +1 -1
  47. package/package.json +20 -6
  48. package/schema/config.json +1 -1
  49. package/skills/branch-review/SKILL.md +123 -0
  50. package/skills/ci-repair/SKILL.md +79 -35
  51. package/skills/garden/SKILL.md +8 -8
  52. package/skills/garden/references/telemetry.md +4 -4
  53. package/skills/incident/SKILL.md +9 -9
  54. package/skills/lens-import/SKILL.md +7 -7
  55. package/skills/review-slop/SKILL.md +1 -1
  56. package/skills/shepherd/SKILL.md +43 -41
  57. package/skills/shepherd/references/degradation.md +3 -3
  58. package/skills/shepherd/references/dispatch.md +1 -1
  59. package/skills/shepherd/references/formats.md +5 -5
  60. package/skills/shepherd/references/models.md +2 -2
  61. package/skills/shepherd/references/policy.md +9 -9
  62. package/skills/swarm/SKILL.md +21 -17
  63. package/skills/swarm/references/custom-lenses.md +2 -2
  64. package/skills/swarm/references/formats.md +1 -1
  65. package/skills/triage/SKILL.md +10 -10
  66. package/src/bin.ts +10 -0
  67. package/src/cli.ts +283 -41
  68. package/src/config.ts +26 -25
  69. package/src/doctor.ts +31 -18
  70. package/src/garden-traces.ts +13 -13
  71. package/src/garden.ts +4 -4
  72. package/src/lens.ts +19 -7
  73. package/src/setup.ts +72 -0
  74. package/src/stack.ts +2 -2
  75. package/src/status.ts +80 -0
  76. package/src/summary.ts +52 -9
  77. package/src/trace.ts +4 -4
  78. package/src/working.ts +30 -0
  79. package/templates/shepherd-audit.yml +7 -15
  80. package/templates/shepherd-incident.yml +7 -12
  81. package/templates/shepherd.yml +39 -40
@@ -0,0 +1,102 @@
1
+ // Every setting gate reads, resolved in one place. A non-blank environment variable wins, then the policy on the
2
+ // default branch, then the built-in default. Settings live in the policy file so a change to one is reviewed like
3
+ // any policy change; only the model (SHEPHERD_MODEL) and the OpenAI-compatible endpoint have variables. Secrets never go in the policy: they are env-only, and
4
+ // `shepherd gate config` reports them as set or unset, never their values.
5
+ import type { Policy } from "./policy.ts";
6
+
7
+ /** Credentials gate reads. Env-only, never printed. */
8
+ export const SECRETS = ["ANTHROPIC_API_KEY", "OPENAI_API_KEY", "OPENCODE_API_KEY", "OPENROUTER_API_KEY", "GROQ_API_KEY", "MISTRAL_API_KEY", "DEEPSEEK_API_KEY", "AI_GATEWAY_API_KEY", "AWS_BEARER_TOKEN_BEDROCK", "CLAUDE_CODE_OAUTH_TOKEN", "CODEX_API_KEY", "TYPESAFE_API_KEY", "OPENAI_COMPATIBLE_API_KEY", "OTEL_EXPORTER_OTLP_HEADERS", "SHEPHERD_NOTIFY_WEBHOOK", "GH_TOKEN"];
9
+
10
+ type Env = Record<string, string | undefined>;
11
+
12
+ export type Settings = Pick<Policy, "model" | "backends" | "pricing" | "provider" | "guard" | "max_output_tokens" | "effort" | "cli" | "label" | "notify" | "telemetry" | "advise_on_deny">;
13
+
14
+ export type GuardSettings = NonNullable<Settings["guard"]>;
15
+
16
+ /** autotel's guard shorthand, in the order gate has always written it. */
17
+ export function guardShorthand(g: GuardSettings): string {
18
+ return [
19
+ g.budget_usd !== undefined && `budget:$${g.budget_usd}`,
20
+ g.tokens !== undefined && `tokens:${g.tokens}`,
21
+ g.loop !== undefined && `loop:${g.loop}`,
22
+ g.max_tools !== undefined && `max-tools:${g.max_tools}`,
23
+ g.timeout !== undefined && `timeout:${g.timeout}`,
24
+ ]
25
+ .filter(Boolean)
26
+ .join(",");
27
+ }
28
+
29
+ export const DEFAULT_GUARD: GuardSettings = { budget_usd: 2, tokens: "3m", loop: "4/12", max_tools: 80, timeout: "15m" };
30
+
31
+ /** Each setting: its policy key (as `shepherd gate config` shows it), the variables that override it, its value in a policy, and its default. "" is unset. */
32
+ const SETTINGS = {
33
+ model: { key: "model", env: ["SHEPHERD_MODEL"], from: (p: Settings) => p.model, fallback: "claude-opus-5" },
34
+ backends: { key: "backends", env: [], from: (p: Settings) => p.backends?.join(","), fallback: "api" },
35
+ pricing: { key: "pricing", env: [], from: (p: Settings) => p.pricing && JSON.stringify(p.pricing), fallback: "" },
36
+ region: { key: "provider.region", env: ["AWS_REGION"], from: (p: Settings) => p.provider?.region, fallback: "" },
37
+ base_url: { key: "provider.base_url", env: ["OPENAI_COMPATIBLE_BASE_URL"], from: (p: Settings) => p.provider?.base_url, fallback: "" },
38
+ anthropic_base_url: { key: "provider.anthropic_base_url", env: ["ANTHROPIC_BASE_URL"], from: (p: Settings) => p.provider?.anthropic_base_url, fallback: "" },
39
+ // A partial `guard:` keeps the default for every rule it leaves out.
40
+ guard: { key: "guard", env: [], from: (p: Settings) => p.guard && guardShorthand({ ...DEFAULT_GUARD, ...p.guard }), fallback: guardShorthand(DEFAULT_GUARD) },
41
+ max_output_tokens: { key: "max_output_tokens", env: [], from: (p: Settings) => p.max_output_tokens?.toString(), fallback: "" },
42
+ effort: { key: "effort", env: [], from: (p: Settings) => p.effort, fallback: "" },
43
+ claude_model: { key: "cli.claude_model", env: [], from: (p: Settings) => p.cli?.claude_model, fallback: "" },
44
+ codex_model: { key: "cli.codex_model", env: [], from: (p: Settings) => p.cli?.codex_model, fallback: "" },
45
+ label: { key: "label", env: [], from: (p: Settings) => p.label, fallback: "" },
46
+ notify_on: { key: "notify.on", env: [], from: (p: Settings) => p.notify?.on?.join(","), fallback: "REFUSED,ESCALATE,ERROR" },
47
+ telemetry_endpoint: { key: "telemetry.endpoint", env: ["OTEL_EXPORTER_OTLP_ENDPOINT"], from: (p: Settings) => p.telemetry?.endpoint, fallback: "" },
48
+ telemetry_service: { key: "telemetry.service", env: ["OTEL_SERVICE_NAME"], from: (p: Settings) => p.telemetry?.service, fallback: "shepherd-gate" },
49
+ advise_on_deny: { key: "advise_on_deny", env: [], from: (p: Settings) => p.advise_on_deny?.toString(), fallback: "false" },
50
+ };
51
+
52
+ export type Setting = keyof typeof SETTINGS;
53
+
54
+ export type Resolved = { value: string; source: "env" | "policy" | "default"; key: string; env: string };
55
+
56
+ /** One setting: the first non-blank variable, else the policy's value, else the default. */
57
+ export function resolve(name: Setting, policy: Settings = {}, env: Env = process.env): Resolved {
58
+ const s = SETTINGS[name];
59
+ const fromEnv = s.env.find((k) => env[k]?.trim());
60
+
61
+ if (fromEnv) return { value: env[fromEnv]!.trim(), source: "env", key: s.key, env: fromEnv }; // SAFETY: find() matched a non-blank env[fromEnv]
62
+
63
+ const fromPolicy = s.from(policy);
64
+
65
+ return fromPolicy !== undefined && fromPolicy !== "" ? { value: fromPolicy, source: "policy", key: s.key, env: s.env[0] ?? "" } : { value: s.fallback, source: "default", key: s.key, env: s.env[0] ?? "" };
66
+ }
67
+
68
+ let active: Settings = {};
69
+
70
+ /** Make the trusted policy's settings the ones every later read sees. Until then, reads see the environment and the defaults. */
71
+ export function useSettings(policy: Settings): void {
72
+ active = policy;
73
+ }
74
+
75
+ /** A setting's effective value, read when it is used, so a variable set later still wins. "" is unset. */
76
+ export const setting = (name: Setting): string => resolve(name, active).value;
77
+
78
+ /** Every setting, with where it came from, and every secret as set or unset. What `shepherd gate config` prints. */
79
+ export function effectiveConfig(policy: Settings = active, env: Env = process.env) {
80
+ // A URL's user:password is a credential: shown redacted, like the secrets below. Up to the authority's last `@`,
81
+ // so a password holding a raw `@` is redacted whole.
82
+ const shown = (name: Setting) => {
83
+ const r = resolve(name, policy, env);
84
+
85
+ return { ...r, value: r.value.replace(/\/\/[^/\s]*@/, "//[redacted]@") };
86
+ };
87
+
88
+ // SAFETY: Object.keys of the SETTINGS literal is exactly Setting.
89
+ const names = Object.keys(SETTINGS) as Setting[];
90
+
91
+ return {
92
+ settings: Object.fromEntries(names.map((k) => [k, shown(k)])),
93
+ secrets: Object.fromEntries(SECRETS.map((k) => [k, env[k]?.trim() ? "set" : "unset"])),
94
+ };
95
+ }
96
+
97
+ /** One line naming every setting that did not come from its default, for the start of a review log. */
98
+ export function configSummary(policy: Settings = active, env: Env = process.env): string {
99
+ const changed = Object.values(effectiveConfig(policy, env).settings).filter((r) => r.source !== "default");
100
+
101
+ return `config: ${changed.length ? changed.map((r) => `${r.key} from ${r.source === "env" ? r.env : "policy"}`).join(", ") : "all defaults"} (shepherd gate config shows every value)`;
102
+ }
@@ -0,0 +1,29 @@
1
+ // What changed between the head gate last reviewed and the head it reviews now. The reviewer gets it as context beside
2
+ // the whole diff, without the last verdict.
3
+ //
4
+ // The gate merges the earlier version onto the new version's fork point and diffs that tree against the new head, which
5
+ // leaves the base branch's own changes out. `merge-tree` writes objects only, and no PR code runs.
6
+ import { execFileSync } from "node:child_process";
7
+ import { callTimeout } from "./github.ts";
8
+
9
+ export type Version = { head: string; base: string };
10
+
11
+ export type Delta = { status: "ok"; diff: string } | { status: "empty" } | { status: "unavailable"; reason: string };
12
+
13
+ /** The net change from `before` to `after`, both already in the local object store. A failure is a status, never a throw. */
14
+ export function incrementalDiff(root: string, before: Version, after: Version): Delta {
15
+ const git = (...args: string[]) => execFileSync("git", ["-C", root, ...args], { encoding: "utf8", maxBuffer: 64 << 20, timeout: callTimeout(), stdio: ["ignore", "pipe", "pipe"] });
16
+ const forkPoint = (v: Version) => git("merge-base", v.head, v.base).trim();
17
+
18
+ try {
19
+ const oldFork = forkPoint(before);
20
+ const newFork = forkPoint(after);
21
+ // On one fork point the heads compare directly. `merge-tree` exits 1 on a conflict, which the catch reports.
22
+ const earlier = oldFork === newFork ? before.head : git("merge-tree", "--write-tree", `--merge-base=${oldFork}`, newFork, before.head).split("\n")[0]!; // SAFETY: split always yields a first element
23
+ const diff = git("diff", "--no-color", "--no-ext-diff", earlier, after.head);
24
+
25
+ return diff.trim() ? { status: "ok", diff } : { status: "empty" };
26
+ } catch (e) {
27
+ return { status: "unavailable", reason: e instanceof Error ? e.message.split("\n")[0]! : String(e) }; // SAFETY: as above
28
+ }
29
+ }
@@ -0,0 +1,30 @@
1
+ // Imported first by cli.ts, ahead of every provider SDK. A workflow passes an unset secret or variable as "",
2
+ // which providers read as a value: @ai-sdk/anthropic builds its default client when the module loads and throws
3
+ // on ANTHROPIC_BASE_URL="", and a blank AWS_BEARER_TOKEN_BEDROCK would replace the OIDC credentials with an
4
+ // empty key. ES modules evaluate in import order, so dropping them here runs before any SDK module does.
5
+
6
+ /** Every variable a provider reads its credentials or endpoint from. The workflow template passes each one. */
7
+ export const PROVIDER_ENV = ["OPENAI_COMPATIBLE_API_KEY", "OPENAI_COMPATIBLE_BASE_URL", "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "OPENAI_API_KEY", "OPENCODE_API_KEY", "OPENROUTER_API_KEY", "GROQ_API_KEY", "MISTRAL_API_KEY", "DEEPSEEK_API_KEY", "AI_GATEWAY_API_KEY", "AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION"];
8
+
9
+ /** C0 controls and DEL: the bytes an HTTP header value cannot carry. */
10
+ const hasControl = (s: string) => [...s].some((c) => c.charCodeAt(0) < 0x20 || c.charCodeAt(0) === 0x7f);
11
+
12
+ /**
13
+ * Drops blank variables and trims the rest, so a key pasted with its newline reaches the provider as a valid header.
14
+ * A key holding a line break goes too, with the reason, and the missing-key path names the fix.
15
+ */
16
+ export function dropBlankProviderEnv(): void {
17
+ for (const k of [...PROVIDER_ENV, "CODEX_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"]) {
18
+ const value = process.env[k]?.trim();
19
+
20
+ if (value === undefined) continue;
21
+
22
+ if (value === "") delete process.env[k];
23
+ else if (hasControl(value)) {
24
+ console.error(`${k} holds a line break or control character and cannot be sent as a header: ignoring it. Save the secret as one line.`);
25
+ delete process.env[k];
26
+ } else process.env[k] = value;
27
+ }
28
+ }
29
+
30
+ dropBlankProviderEnv();
@@ -0,0 +1,288 @@
1
+ // The review benchmark: frozen changes with known outcomes, reviewed through gate's own path, scored for the
2
+ // verdicts that matter to an approval bot. A case is a plain git repository (`export` writes one), so any other
3
+ // reviewer can review the same change and be scored from run records in the same shape.
4
+ //
5
+ // bun run eval run evals/cases --label nova --repeat 3 (SHEPHERD_MODEL and the provider key as for a review; --case one)
6
+ // bun run eval score evals/runs
7
+ // bun run eval export evals/cases/off-by-one /tmp/off-by-one (a repository: base on main, the change on pr)
8
+ import { execFileSync } from "node:child_process";
9
+ import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
10
+ import { tmpdir } from "node:os";
11
+ import path from "node:path";
12
+ import { z } from "zod";
13
+ import { setting } from "./config.ts";
14
+ import type { PR } from "./github.ts";
15
+ import { leadsAt } from "./leads.ts";
16
+ import { assured, diffCut, readAll, review, reviewers, type Backend } from "./reviewer.ts";
17
+
18
+ // A file's content: text, or a line repeated (`{ "repeat": "x = 1", "times": 20000 }`) for a diff past the prompt limit.
19
+ const Content = z.union([z.string(), z.object({ repeat: z.string(), times: z.number().int().positive() }).transform((r) => `${Array.from({ length: r.times }, () => r.repeat).join("\n")}\n`)]);
20
+
21
+ /** A case path stays inside its repository: relative, no `..`, nothing under `.git`, where a hook would run. */
22
+ const CasePath = z.string().refine((p) => {
23
+ const parts = p.split(/[\\/]/);
24
+
25
+ return p !== "" && !path.isAbsolute(p) && !/^[a-z]:/i.test(p) && !parts.includes("..") && !parts.some((s) => s.toLowerCase() === ".git");
26
+ }, "a case path is relative, inside the repository and outside .git");
27
+
28
+ export const Case = z.object({
29
+ title: z.string(),
30
+ body: z.string().default(""),
31
+ base: z.record(CasePath, Content),
32
+ /** The change: new content per path, null to delete. Paths not named keep their base content. */
33
+ head: z.record(CasePath, Content.nullable()),
34
+ /** `block`: a safe reviewer does not approve. `approve`: nothing in it should hold an approval back. */
35
+ expect: z.enum(["approve", "block"]),
36
+ /** The bugs a person confirmed: credited when one issue names the file and matches a pattern. */
37
+ bugs: z.array(z.object({ id: z.string(), file: z.string(), match: z.array(z.string()).min(1) })).default([]),
38
+ split: z.enum(["dev", "held-out"]).default("dev"),
39
+ });
40
+
41
+ export type Case = z.infer<typeof Case>;
42
+
43
+ /** One review of one case, by any reviewer. `verdict` is null when the run failed. */
44
+ export const Run = z.object({
45
+ case: z.string(),
46
+ label: z.string(),
47
+ verdict: z.enum(["APPROVE", "REFUSE", "ESCALATE"]).nullable(),
48
+ /** The model's own verdict, before gate's approval checks (assured, readAll): where they differ, gate held it. */
49
+ model_verdict: z.enum(["APPROVE", "REFUSE", "ESCALATE"]).nullable().optional(),
50
+ issues: z.array(z.string()),
51
+ costUsd: z.number().nullable(),
52
+ durationMs: z.number(),
53
+ error: z.string().optional(),
54
+ });
55
+
56
+ export type Run = z.infer<typeof Run>;
57
+
58
+ const git = (dir: string, ...args: string[]) =>
59
+ execFileSync("git", ["-C", dir, "-c", "core.hooksPath=/dev/null", "-c", "user.name=eval", "-c", "user.email=eval@localhost", "-c", "core.quotepath=true", ...args], { encoding: "utf8", maxBuffer: 256 << 20 }).trim();
60
+
61
+ /** A materialized case: the PR's diff and its changed files, as gate reads them from GitHub. */
62
+ export type Change = { diff: string; files: PR["files"] };
63
+
64
+ /** The case as a repository in `dir`: its base committed on `main`, the change on `pr`. */
65
+ export function materialize(c: Case, dir: string): Change {
66
+ const put = (files: Record<string, string | null>) => {
67
+ for (const [p, content] of Object.entries(files)) {
68
+ const at = path.resolve(dir, p);
69
+
70
+ // The schema checked the spelling; this checks where it lands.
71
+ if (!at.startsWith(path.resolve(dir) + path.sep)) throw new Error(`case path escapes the repository: ${p}`);
72
+
73
+ if (content === null) rmSync(at, { force: true });
74
+ else {
75
+ mkdirSync(path.dirname(at), { recursive: true });
76
+ writeFileSync(at, content);
77
+ }
78
+ }
79
+
80
+ git(dir, "add", "-A");
81
+ };
82
+
83
+ mkdirSync(dir, { recursive: true });
84
+ git(dir, "init", "-q", "-b", "main");
85
+ put(c.base);
86
+ git(dir, "commit", "-qm", "base");
87
+ git(dir, "checkout", "-qb", "pr");
88
+ put(c.head);
89
+ git(dir, "commit", "-qm", c.title);
90
+
91
+ return { diff: `${git(dir, "diff", "-M", "main", "pr")}\n`, files: changedFiles(git(dir, "diff", "-M", "--numstat", "-z", "main", "pr"), git(dir, "diff", "-M", "--name-status", "-z", "main", "pr")) };
92
+ }
93
+
94
+ const STATUS = new Map([["A", "added"], ["D", "removed"], ["R", "renamed"], ["C", "copied"]]);
95
+
96
+ /**
97
+ * The PR's files as GitHub's API lists them, from `git diff -z` output. A rename or copy spans tokens: numstat
98
+ * writes `add\tdel\t` then the old and new paths as tokens of their own, name-status writes `R100` then both.
99
+ */
100
+ export function changedFiles(numstat: string, nameStatus: string): PR["files"] {
101
+ const status = new Map<string, string>();
102
+ const st = nameStatus.split("\0");
103
+
104
+ for (let i = 0; i + 1 < st.length; ) {
105
+ const code = st[i]![0]!;
106
+
107
+ if (code === "R" || code === "C") {
108
+ status.set(st[i + 2]!, STATUS.get(code)!);
109
+ i += 3;
110
+ } else {
111
+ status.set(st[i + 1]!, STATUS.get(code) ?? "modified");
112
+ i += 2;
113
+ }
114
+ }
115
+
116
+ const files: PR["files"] = [];
117
+ const ns = numstat.split("\0");
118
+
119
+ for (let i = 0; i < ns.length; i++) {
120
+ // `add\tdel\tpath`: split at the first two tabs only, since a path may hold tabs of its own.
121
+ const [, add = "", del = "", path = ""] = /^([^\t]*)\t([^\t]*)\t?([\s\S]*)$/.exec(ns[i]!) ?? [];
122
+
123
+ if (!add) continue;
124
+ // Binary files count "-" for both: no lines to size.
125
+ const counts = { additions: Number(add) || 0, deletions: Number(del) || 0 };
126
+
127
+ if (path) {
128
+ files.push({ filename: path, ...counts, status: status.get(path) ?? "modified" });
129
+ } else {
130
+ const [previousFilename = "", filename = ""] = [ns[i + 1], ns[i + 2]];
131
+
132
+ files.push({ filename, previousFilename, ...counts, status: status.get(filename) ?? "renamed" });
133
+ i += 2;
134
+ }
135
+ }
136
+
137
+ return files;
138
+ }
139
+
140
+
141
+ export function loadCases(dir: string): [string, Case][] {
142
+ return readdirSync(dir, { withFileTypes: true })
143
+ .filter((d) => d.isDirectory())
144
+ .map((d): [string, Case] => [d.name, Case.parse(JSON.parse(readFileSync(path.join(dir, d.name, "case.json"), "utf8")))])
145
+ .sort(([a], [b]) => a.localeCompare(b));
146
+ }
147
+
148
+ /** One review through gate's own path: the prompt, leads and read tools a PR gets, then the approval checks the CLI applies. */
149
+ export async function runCase(name: string, c: Case, backend: Backend, label: string, guidance: string): Promise<Run> {
150
+ const dir = mkdtempSync(path.join(tmpdir(), `gate-eval-${name}-`));
151
+ const started = Date.now();
152
+
153
+ try {
154
+ const { diff, files } = materialize(c, dir);
155
+ const size = files.reduce((n, f) => n + f.additions + f.deletions, 0);
156
+
157
+ const pr: PR = {
158
+ repo: "eval/case", number: 1, title: c.title, body: c.body, author: "eval", authorAssociation: "MEMBER", isFork: false, isDraft: false, mergeable: "MERGEABLE",
159
+ baseRef: "main", headRef: "pr", defaultBranch: "main", headSha: git(dir, "rev-parse", "pr"), baseSha: git(dir, "rev-parse", "main"),
160
+ labels: [], files, reviews: [], inline: [], discussion: [], reactions: [], commits: [], diff,
161
+ };
162
+
163
+ const input = {
164
+ pr,
165
+ gates: [{ gate: "size", passed: true, message: `${size} lines / ${files.length} files` }],
166
+ gateVerdict: "PASSED" as const,
167
+ tier: "T1-agent",
168
+ titleFlags: [],
169
+ manifests: [],
170
+ scrutiny: [],
171
+ leads: leadsAt(dir, diff, files.flatMap((f) => [f.filename, ...(f.previousFilename ? [f.previousFilename] : [])])),
172
+ };
173
+
174
+ const llm = await review(backend, input, guidance, dir);
175
+ const outcome = readAll(assured({ verdict: llm.verdict, llm }), diffCut(diff), [llm]);
176
+
177
+ return { case: name, label, verdict: outcome.verdict, model_verdict: llm.verdict, issues: outcome.llm.issues, costUsd: llm.run?.costUsd ?? null, durationMs: Date.now() - started };
178
+ } catch (e) {
179
+ return { case: name, label, verdict: null, issues: [], costUsd: null, durationMs: Date.now() - started, error: String(e instanceof Error ? e.message : e).split("\n")[0] };
180
+ } finally {
181
+ rmSync(dir, { recursive: true, force: true });
182
+ }
183
+ }
184
+
185
+ /** Bug ids an issue list finds: an issue naming the bug's file (or its basename) that matches one of its patterns. */
186
+ export function found(c: Case, issues: string[]): string[] {
187
+ return c.bugs.flatMap((b) =>
188
+ issues.some((i) => (i.includes(b.file) || i.includes(path.basename(b.file))) && b.match.some((m) => new RegExp(m, "i").test(i))) ? [b.id] : [],
189
+ );
190
+ }
191
+
192
+ export type Score = {
193
+ label: string;
194
+ runs: number;
195
+ errors: number;
196
+ /** Approvals of a `block` case: the failure an approval bot must not have. */
197
+ unsafe_approvals: number;
198
+ /** `approve` cases held back: a cost to the team, never a safety failure. */
199
+ false_holds: number;
200
+ bug_recall: number | null;
201
+ /** Issues on `block` cases that find no listed bug, and every issue on an `approve` case. Check them: one may be a bug nobody listed. */
202
+ unmatched_issues: number;
203
+ /** Runs whose model approved and gate's checks held back: what the checks add over the model alone. */
204
+ held_by_gate: number;
205
+ cost_usd: number | null;
206
+ median_ms: number;
207
+ };
208
+
209
+ export function score(cases: Map<string, Case>, runs: Run[]): Score[] {
210
+ const labels = [...new Set(runs.map((r) => r.label))].sort();
211
+
212
+ return labels.map((label) => {
213
+ const mine = runs.filter((r) => r.label === label && cases.has(r.case));
214
+ const done = mine.filter((r) => r.verdict !== null);
215
+ let bugs = 0;
216
+ let hits = 0;
217
+ let unmatched = 0;
218
+
219
+ for (const r of done) {
220
+ const c = cases.get(r.case)!;
221
+ const ids = found(c, r.issues);
222
+
223
+ bugs += c.bugs.length;
224
+ hits += ids.length;
225
+ unmatched += c.expect === "approve" ? r.issues.length : r.issues.filter((i) => !found(c, [i]).length).length;
226
+ }
227
+
228
+ const ms = mine.map((r) => r.durationMs).sort((a, b) => a - b);
229
+ const costs = mine.map((r) => r.costUsd);
230
+
231
+ return {
232
+ label,
233
+ runs: mine.length,
234
+ errors: mine.length - done.length,
235
+ unsafe_approvals: done.filter((r) => r.verdict === "APPROVE" && cases.get(r.case)!.expect === "block").length,
236
+ false_holds: done.filter((r) => r.verdict !== "APPROVE" && cases.get(r.case)!.expect === "approve").length,
237
+ bug_recall: bugs ? hits / bugs : null,
238
+ unmatched_issues: unmatched,
239
+ held_by_gate: done.filter((r) => r.model_verdict === "APPROVE" && r.verdict !== "APPROVE").length,
240
+ cost_usd: costs.every((c) => c !== null) ? costs.reduce<number>((n, c) => n + c!, 0) : null,
241
+ median_ms: ms[Math.floor(ms.length / 2)] ?? 0,
242
+ };
243
+ });
244
+ }
245
+
246
+ const readRuns = (dir: string): Run[] =>
247
+ readdirSync(dir, { recursive: true, withFileTypes: true }).flatMap((f) => (f.isFile() && f.name.endsWith(".json") ? [Run.parse(JSON.parse(readFileSync(path.join(f.parentPath, f.name), "utf8")))] : []));
248
+
249
+ if (import.meta.main) {
250
+ const [command, a, b] = process.argv.slice(2);
251
+
252
+ const flag = (name: string, fallback: string) => {
253
+ const i = process.argv.indexOf(`--${name}`);
254
+
255
+ return i > 0 ? process.argv[i + 1]! : fallback;
256
+ };
257
+
258
+ if (command === "run" && a) {
259
+ const label = flag("label", setting("model"));
260
+ const repeat = Number(flag("repeat", "1"));
261
+ const out = path.join(flag("out", "evals/runs"), label, new Date().toISOString().replace(/[:.]/g, "-"));
262
+ const guidance = readFileSync(path.resolve(import.meta.dir, "../.shepherd/review-guidance.md"), "utf8");
263
+
264
+ mkdirSync(out, { recursive: true });
265
+
266
+ const only = flag("case", "");
267
+
268
+ for (const [name, c] of loadCases(a).filter(([n]) => !only || n === only))
269
+ for (let n = 1; n <= repeat; n++) {
270
+ const run = await runCase(name, c, reviewers()[0]!, label, guidance);
271
+
272
+ writeFileSync(path.join(out, `${name}-${n}.json`), `${JSON.stringify(run, null, 2)}\n`);
273
+ console.log(`${name} #${n}: ${run.verdict ?? `ERROR ${run.error}`} (${c.expect}) ${run.costUsd === null ? "unpriced" : `$${run.costUsd.toFixed(4)}`} ${(run.durationMs / 1000).toFixed(1)}s`);
274
+ }
275
+
276
+ console.log(`runs in ${out}`);
277
+ } else if (command === "score" && a) {
278
+ const cases = new Map(loadCases(flag("cases", "evals/cases")));
279
+
280
+ console.table(score(cases, readRuns(a)));
281
+ } else if (command === "export" && a && b) {
282
+ materialize(Case.parse(JSON.parse(readFileSync(path.join(a, "case.json"), "utf8"))), b);
283
+ console.log(`${b}: base on main, the change on pr`);
284
+ } else {
285
+ console.error("usage: eval run <cases> [--label <l>] [--repeat <n>] [--case <name>] [--out <dir>] | eval score <runs> [--cases <dir>] | eval export <case> <dir>");
286
+ process.exit(2);
287
+ }
288
+ }