@jagreehal/shepherd 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -5
- package/README.md +40 -27
- package/THIRD_PARTY_NOTICES.md +2 -0
- package/gate/.shepherd/gate.schema.json +647 -0
- package/gate/.shepherd/gate.yml +122 -0
- package/gate/.shepherd/review-guidance.md +95 -0
- package/gate/src/actions.ts +32 -0
- package/gate/src/cli.ts +1028 -0
- package/gate/src/config.ts +102 -0
- package/gate/src/delta.ts +29 -0
- package/gate/src/env.ts +30 -0
- package/gate/src/eval.ts +288 -0
- package/gate/src/familiarity.ts +461 -0
- package/gate/src/github.ts +761 -0
- package/gate/src/inline.ts +59 -0
- package/gate/src/isolated-cli.ts +281 -0
- package/gate/src/leads.ts +257 -0
- package/gate/src/llm.ts +606 -0
- package/gate/src/notify.ts +57 -0
- package/gate/src/policy.ts +1297 -0
- package/gate/src/provider-errors.ts +80 -0
- package/gate/src/retention.ts +86 -0
- package/gate/src/reviewer.ts +739 -0
- package/gate/src/signals.ts +105 -0
- package/gate/src/smoke-live.ts +64 -0
- package/gate/src/stack.ts +87 -0
- package/gate/src/stories.ts +297 -0
- package/gate/templates/policy-strict.yml +127 -0
- package/gate/templates/reviewer.Dockerfile +8 -0
- package/gate/templates/rules/observability/SKILL.md +33 -0
- package/gate/templates/shepherd-gate-actions.yml +35 -0
- package/gate/templates/shepherd-gate-edited.yml +29 -0
- package/gate/templates/shepherd-gate.yml +111 -0
- package/lenses/dotnet/SKILL.md +1 -1
- package/lenses/go/SKILL.md +1 -1
- package/lenses/java/SKILL.md +1 -1
- package/lenses/nextjs/SKILL.md +1 -1
- package/lenses/node-api/SKILL.md +1 -1
- package/lenses/nx-monorepo/SKILL.md +1 -1
- package/lenses/observability/SKILL.md +1 -1
- package/lenses/owasp-asvs/SKILL.md +1 -1
- package/lenses/owasp-llm/SKILL.md +1 -1
- package/lenses/pstack/SKILL.md +1 -1
- package/lenses/python/SKILL.md +1 -1
- package/lenses/react/SKILL.md +1 -1
- package/lenses/rust/SKILL.md +1 -1
- package/package.json +20 -6
- package/schema/config.json +1 -1
- package/skills/branch-review/SKILL.md +123 -0
- package/skills/ci-repair/SKILL.md +79 -35
- package/skills/garden/SKILL.md +29 -10
- package/skills/garden/references/telemetry.md +4 -4
- package/skills/incident/SKILL.md +19 -9
- package/skills/lens-import/SKILL.md +7 -7
- package/skills/review-maintainability/SKILL.md +1 -1
- package/skills/review-slop/SKILL.md +1 -1
- package/skills/shepherd/SKILL.md +59 -44
- package/skills/shepherd/references/degradation.md +3 -3
- package/skills/shepherd/references/dispatch.md +2 -2
- package/skills/shepherd/references/formats.md +5 -5
- package/skills/shepherd/references/models.md +2 -2
- package/skills/shepherd/references/policy.md +9 -9
- package/skills/swarm/SKILL.md +26 -17
- package/skills/swarm/references/custom-lenses.md +2 -2
- package/skills/swarm/references/formats.md +1 -1
- package/skills/triage/SKILL.md +11 -11
- package/src/bin.ts +10 -0
- package/src/cli.ts +283 -41
- package/src/config.ts +26 -25
- package/src/doctor.ts +31 -18
- package/src/garden-traces.ts +13 -13
- package/src/garden.ts +43 -8
- package/src/lens.ts +19 -7
- package/src/setup.ts +72 -0
- package/src/stack.ts +2 -2
- package/src/status.ts +80 -0
- package/src/summary.ts +52 -9
- package/src/trace.ts +4 -4
- package/src/working.ts +30 -0
- package/templates/shepherd-audit.yml +7 -15
- package/templates/shepherd-incident.yml +9 -12
- package/templates/shepherd.yml +39 -40
package/src/doctor.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { parse } from "yaml";
|
|
|
6
6
|
import { z } from "zod";
|
|
7
7
|
import { defaultBranchReader, defaultBranchRef, fetchPackLenses, ghFetch, ghFetchDir, loadPolicy, type Config } from "./config.ts";
|
|
8
8
|
import { bundledSkills } from "./install.ts";
|
|
9
|
-
import { lensProblems, pluginRoots, skillDirs } from "./lens.ts";
|
|
9
|
+
import { lensProblems, pluginRoots, providerKeyEnv, skillDirs } from "./lens.ts";
|
|
10
10
|
|
|
11
11
|
export type Check = { check: string; status: "ok" | "warn" | "fail"; detail: string; fix?: string };
|
|
12
12
|
|
|
@@ -24,20 +24,22 @@ const attempt = <T>(f: () => T): T | Error => {
|
|
|
24
24
|
const SKILL_DIRS = (home: string, repo: string) => [...skillDirs(home, repo), path.join(home, ".config/opencode/skills")];
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
|
-
* True when a workflow runs
|
|
28
|
-
*
|
|
27
|
+
* True when a workflow runs the gate on pull requests: a step that runs `shepherd gate` from the package, or from a
|
|
28
|
+
* checkout of shepherd's repository. The word "gate" alone is not enough.
|
|
29
29
|
*/
|
|
30
|
-
export function
|
|
30
|
+
export function runsGate(text: string): boolean {
|
|
31
31
|
const parsed = Workflow.safeParse(attempt(() => parse(text)));
|
|
32
32
|
|
|
33
33
|
if (!parsed.success) return false;
|
|
34
34
|
|
|
35
35
|
if (!parsed.data.on.some((t) => t === "pull_request" || t === "pull_request_target")) return false;
|
|
36
36
|
|
|
37
|
+
const gateCall = /(@jagreehal\/shepherd\S*|\bshepherd|\/src\/bin\.ts)"?\s+gate\b/;
|
|
38
|
+
|
|
37
39
|
return Object.values(parsed.data.jobs).some(({ steps }) => {
|
|
38
|
-
const checkouts = steps.filter((st) => st.with?.repository === "jagreehal/
|
|
40
|
+
const checkouts = steps.filter((st) => st.with?.repository === "jagreehal/shepherd").map((st) => st.with?.path ?? ".");
|
|
39
41
|
|
|
40
|
-
return steps.some((st) => st.
|
|
42
|
+
return steps.some((st) => gateCall.test(st.run ?? "") && (st.run?.includes("@jagreehal/shepherd") || /\bshepherd\s+gate\b/.test(st.run ?? "") || checkouts.some((p) => p !== "." && st.run?.includes(`${p}/`))));
|
|
41
43
|
});
|
|
42
44
|
}
|
|
43
45
|
|
|
@@ -128,28 +130,39 @@ export function doctor(bundleRoot: string, home: string, repo: string, personal:
|
|
|
128
130
|
if (!models.length) add("models", "ok", "no ladder set: the harness's own (shepherd models)");
|
|
129
131
|
|
|
130
132
|
const workflows = attempt(() => run("git", ["-C", repo, "ls-tree", "--name-only", "-r", defaultBranchRef(repo), ".github/workflows/"]));
|
|
131
|
-
const
|
|
133
|
+
const gate = (workflows instanceof Error ? [] : workflows.split("\n").filter(Boolean)).find((f) => runsGate(read(f) ?? ""));
|
|
132
134
|
|
|
133
|
-
if (!
|
|
134
|
-
add("
|
|
135
|
+
if (!gate) {
|
|
136
|
+
add("gate workflow", "warn", "no default-branch workflow runs the gate on pull requests: shepherd can review and fix, but nothing gives the approval verdict", "shepherd workflows, then add the model key");
|
|
135
137
|
|
|
136
138
|
return checks;
|
|
137
139
|
}
|
|
138
140
|
|
|
139
|
-
add("
|
|
140
|
-
|
|
141
|
+
add("gate workflow", "ok", `${gate} runs the gate on pull requests`);
|
|
142
|
+
// The first review fails on a missing key, so check for it before anyone opens a PR. An environment's secrets are
|
|
143
|
+
// not listed here, hence a warning.
|
|
144
|
+
const loop = policy.models.loop?.value ?? "";
|
|
145
|
+
const key = loop ? providerKeyEnv(loop) : null;
|
|
146
|
+
const secrets = key ? attempt(() => run("gh", ["secret", "list", "--json", "name", "--jq", ".[].name"], repo)) : null;
|
|
147
|
+
|
|
148
|
+
if (key && !(secrets instanceof Error) && secrets !== null) {
|
|
149
|
+
if (secrets.split("\n").includes(key)) add("model key", "ok", `${key} is set for ${loop}`);
|
|
150
|
+
else add("model key", "warn", `${key} is not a repository secret, so ${loop} cannot run (fine if a GitHub environment holds it)`, `gh secret set ${key}`);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const last = attempt(() => run("gh", ["run", "list", "--workflow", path.basename(gate), "--limit", "30", "--json", "conclusion,status,url,databaseId", "--jq", "[.[] | select(.conclusion != \"skipped\")][0] // empty | [.status, .conclusion, .url, .databaseId] | @tsv"], repo).trim());
|
|
141
154
|
|
|
142
|
-
if (last instanceof Error) add("
|
|
143
|
-
else if (!last) add("
|
|
155
|
+
if (last instanceof Error) add("gate runs", "warn", "cannot read its runs through gh", "check gh can read this repository's Actions");
|
|
156
|
+
else if (!last) add("gate runs", "warn", "configured, but no recent run did anything (all skipped, or none)", "open a PR, or comment /gate on one, and check the run");
|
|
144
157
|
else {
|
|
145
158
|
const [status, conclusion, url, id] = last.split("\t");
|
|
146
|
-
//
|
|
159
|
+
// gate exits 1 on a refusal or an escalation: the run failed, but gate worked and gave its verdict.
|
|
147
160
|
const verdict = conclusion === "failure" ? /\b(REFUSED|ESCALATE): /.exec(String(attempt(() => run("gh", ["run", "view", id!, "--log-failed"], repo))))?.[1] : undefined;
|
|
148
161
|
|
|
149
|
-
if (conclusion === "success") add("
|
|
150
|
-
else if (verdict) add("
|
|
151
|
-
else if (conclusion === "action_required") add("
|
|
152
|
-
else add("
|
|
162
|
+
if (conclusion === "success") add("gate runs", "ok", "its latest run approved");
|
|
163
|
+
else if (verdict) add("gate runs", "ok", `its latest run gave a verdict (${verdict}): ${url}`);
|
|
164
|
+
else if (conclusion === "action_required") add("gate runs", "warn", `its latest run is waiting for a maintainer to approve it: ${url}`, "approve the run in the Actions tab (GitHub holds runs from first-time contributors and bots), or open a PR from a branch of this repository");
|
|
165
|
+
else add("gate runs", "warn", `its latest run is ${conclusion || status}: ${url}`, "read the run log; a missing backend secret is the usual cause (gate README, Backends)");
|
|
153
166
|
}
|
|
154
167
|
|
|
155
168
|
return checks;
|
package/src/garden-traces.ts
CHANGED
|
@@ -13,7 +13,7 @@ const MAX_SPANS = 50_000;
|
|
|
13
13
|
|
|
14
14
|
const MAX_MEASUREMENTS = 50_000_000;
|
|
15
15
|
|
|
16
|
-
const KEYS = new Set(["vcs.repository.name", "vcs.change.id", "vcs.ref.head.revision", "shepherd.retries", "shepherd.stage", "shepherd.outcome", "
|
|
16
|
+
const KEYS = new Set(["vcs.repository.name", "vcs.change.id", "vcs.ref.head.revision", "shepherd.retries", "shepherd.stage", "shepherd.outcome", "gate.cost_usd", "gate.cost_complete"]);
|
|
17
17
|
|
|
18
18
|
const Id = z.string().transform((s) => /^[0-9a-f]{1,32}$/.test(s) ? s.padStart(32, "0") : /^[A-Za-z0-9+/]{22}==$/.test(s) ? Buffer.from(s, "base64").toString("hex") : s)
|
|
19
19
|
.pipe(z.string().regex(/^[0-9a-f]{32}$/).refine((s) => /[1-9a-f]/.test(s)));
|
|
@@ -164,7 +164,7 @@ export async function readTempo(endpoint: string, headers: Record<string, string
|
|
|
164
164
|
const signal = AbortSignal.timeout(30_000);
|
|
165
165
|
const get = async (url: URL) => readResponse(await fetch(url, { headers: { ...headers, Accept: "application/json" }, redirect: "error", signal }));
|
|
166
166
|
const search = new URL("api/search", base);
|
|
167
|
-
search.searchParams.set("q", `{ (name = "shepherd.iteration" || name = "
|
|
167
|
+
search.searchParams.set("q", `{ (name = "shepherd.iteration" || name = "gate.review") && span.vcs.repository.name = ${JSON.stringify(repo)} }`);
|
|
168
168
|
search.searchParams.set("start", String(Math.floor(Date.parse(since) / 1000)));
|
|
169
169
|
search.searchParams.set("end", String(Math.floor(now / 1000)));
|
|
170
170
|
search.searchParams.set("limit", String(MAX_TRACES));
|
|
@@ -197,7 +197,7 @@ const DatadogCustom = z.object({
|
|
|
197
197
|
}).optional().catch(undefined),
|
|
198
198
|
shepherd: z.object({ retries: num, stage: str, outcome: str }).optional().catch(undefined),
|
|
199
199
|
// Datadog stores a boolean attribute as the string "true" or "false".
|
|
200
|
-
|
|
200
|
+
gate: z.object({ cost_usd: num, cost_complete: z.enum(["true", "false"]).optional().catch(undefined) }).optional().catch(undefined),
|
|
201
201
|
otel: z.object({ status_code: str }).optional().catch(undefined),
|
|
202
202
|
}).catch({});
|
|
203
203
|
|
|
@@ -247,8 +247,8 @@ export function fromDatadog(span: DatadogSpan): Span | null {
|
|
|
247
247
|
...asDouble("shepherd.retries", c.shepherd?.retries),
|
|
248
248
|
...asText("shepherd.stage", c.shepherd?.stage),
|
|
249
249
|
...asText("shepherd.outcome", c.shepherd?.outcome),
|
|
250
|
-
...asDouble("
|
|
251
|
-
...(c.
|
|
250
|
+
...asDouble("gate.cost_usd", c.gate?.cost_usd),
|
|
251
|
+
...(c.gate?.cost_complete === undefined ? [] : [{ key: "gate.cost_complete", value: { boolValue: c.gate.cost_complete === "true" } }]),
|
|
252
252
|
];
|
|
253
253
|
|
|
254
254
|
const parsed = Span.safeParse({
|
|
@@ -261,7 +261,7 @@ export function fromDatadog(span: DatadogSpan): Span | null {
|
|
|
261
261
|
}
|
|
262
262
|
|
|
263
263
|
/**
|
|
264
|
-
* Datadog's span search: the repository's `shepherd.iteration` and `
|
|
264
|
+
* Datadog's span search: the repository's `shepherd.iteration` and `gate.review` roots, then every span in those
|
|
265
265
|
* traces, since stage spans carry no repository. The endpoint rate-limits at 300 requests an hour, so the reader
|
|
266
266
|
* batches trace IDs and stops at the trace and span caps.
|
|
267
267
|
*/
|
|
@@ -279,7 +279,7 @@ export async function readDatadog(site: string, apiKey: string, appKey: string,
|
|
|
279
279
|
signal,
|
|
280
280
|
}))));
|
|
281
281
|
|
|
282
|
-
const roots = await search(`@vcs.repository.name:${JSON.stringify(repo)} resource_name:(shepherd.iteration OR
|
|
282
|
+
const roots = await search(`@vcs.repository.name:${JSON.stringify(repo)} resource_name:(shepherd.iteration OR gate.review)`);
|
|
283
283
|
const ids = [...new Set(roots.data.flatMap((s) => (s ? [s.attributes.trace_id] : [])).filter((id) => /^[0-9a-f]{32}$/.test(id)))].slice(0, MAX_TRACES);
|
|
284
284
|
// Here `limited` tracks only the span budget, so a truncated root search still pages through every trace it found.
|
|
285
285
|
const input: TraceInput = { spans: [], sources: 1, unavailable: 0, limited: false };
|
|
@@ -372,10 +372,10 @@ export type TraceSummary = {
|
|
|
372
372
|
reviewMs: number; knownCostUsd: number; pricedReviews: number; unpricedReviews: number;
|
|
373
373
|
stages: { stage: typeof STAGES[number]; durationMs: number; runs: number; failed: number }[];
|
|
374
374
|
signals: TraceSignal[];
|
|
375
|
-
evidence: { traceId: string; spanId: string; headSha: string; kind: "shepherd.iteration" | "
|
|
375
|
+
evidence: { traceId: string; spanId: string; headSha: string; kind: "shepherd.iteration" | "gate.review" }[];
|
|
376
376
|
};
|
|
377
377
|
|
|
378
|
-
const Kind = z.enum(["shepherd.iteration", "
|
|
378
|
+
const Kind = z.enum(["shepherd.iteration", "gate.review"]);
|
|
379
379
|
|
|
380
380
|
/** Deduplicate exports, then correlate only known PR commits inside the requested window. These are observed samples, not a complete history. */
|
|
381
381
|
export function traceScores(input: TraceInput, repo: string, prs: PrRecord[], since: string, limits = traceLimits, now = Date.now()) {
|
|
@@ -409,14 +409,14 @@ export function traceScores(input: TraceInput, repo: string, prs: PrRecord[], si
|
|
|
409
409
|
|
|
410
410
|
if (summary.evidence.length < 20) summary.evidence.push({ traceId: root.traceId, spanId: root.spanId, headSha: head, kind: kind.data });
|
|
411
411
|
|
|
412
|
-
if (kind.data === "
|
|
412
|
+
if (kind.data === "gate.review") {
|
|
413
413
|
summary.reviews++;
|
|
414
414
|
summary.reviewMs += duration(root);
|
|
415
|
-
const cost = number(root, "
|
|
415
|
+
const cost = number(root, "gate.cost_usd");
|
|
416
416
|
|
|
417
417
|
if (cost !== undefined) summary.knownCostUsd += cost;
|
|
418
418
|
|
|
419
|
-
if (cost !== undefined && value(root, "
|
|
419
|
+
if (cost !== undefined && value(root, "gate.cost_complete")?.boolValue === true) summary.pricedReviews++;
|
|
420
420
|
else summary.unpricedReviews++;
|
|
421
421
|
} else {
|
|
422
422
|
summary.iterations++;
|
|
@@ -436,7 +436,7 @@ export function traceScores(input: TraceInput, repo: string, prs: PrRecord[], si
|
|
|
436
436
|
record.failed += Number(failed);
|
|
437
437
|
summary.failedStages += Number(failed);
|
|
438
438
|
|
|
439
|
-
if (stage.data === "
|
|
439
|
+
if (stage.data === "gate") waits.push([startMs(child), startMs(child) + elapsed]);
|
|
440
440
|
|
|
441
441
|
if (!summary.stages.includes(record)) summary.stages.push(record);
|
|
442
442
|
}
|
package/src/garden.ts
CHANGED
|
@@ -25,6 +25,8 @@ export type PrRecord = {
|
|
|
25
25
|
|
|
26
26
|
const DAY = 864e5;
|
|
27
27
|
|
|
28
|
+
type Outcome = "fixed" | "dismissed" | "argued" | "open";
|
|
29
|
+
|
|
28
30
|
export type Signal = "reverted_fix" | "human_replied" | "round_cap" | "stopped_short" | "loop_stalled" | "churn";
|
|
29
31
|
|
|
30
32
|
const TAG = /\*\*\[(?:convergent: )?([^\]]+)\]\*\*\s*\S+\s*(CRITICAL|HIGH|MEDIUM|LOW|NIT)/;
|
|
@@ -81,6 +83,14 @@ export function score(all: PrRecord[], minPrs = 3, now = Date.now()) {
|
|
|
81
83
|
};
|
|
82
84
|
});
|
|
83
85
|
|
|
86
|
+
const reviewed = prs.flatMap((p) => (p.trusted && p.state === "MERGED" && p.rounds ? [p.rounds] : []));
|
|
87
|
+
|
|
88
|
+
const rounds = {
|
|
89
|
+
prs: reviewed.length,
|
|
90
|
+
mean: reviewed.length ? reviewed.reduce((a, b) => a + b, 0) / reviewed.length : null,
|
|
91
|
+
oneRound: reviewed.length ? reviewed.filter((r) => r === 1).length / reviewed.length : null,
|
|
92
|
+
};
|
|
93
|
+
|
|
84
94
|
const byRule = new Map<string, { prs: Set<number>; paths: Set<string> }>();
|
|
85
95
|
|
|
86
96
|
// Accepted means triage fixed it and said so in a Shepherd-Fixes trailer. Triage also resolves findings it
|
|
@@ -104,7 +114,29 @@ export function score(all: PrRecord[], minPrs = 3, now = Date.now()) {
|
|
|
104
114
|
.flatMap(([rule, e]) => (e.prs.size >= minPrs ? [{ rule, prs: [...e.prs].sort((a, b) => a - b), paths: [...e.paths].sort() }] : []))
|
|
105
115
|
.sort((a, b) => b.prs.length - a.prs.length || a.rule.localeCompare(b.rule));
|
|
106
116
|
|
|
107
|
-
return { prs: scored, recurring };
|
|
117
|
+
return { prs: scored, recurring, rounds, outcomes: outcomesOf(prs.filter((p) => p.trusted)) };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Each lens rule's threads by how they ended, noisiest first. Only a rule can match a trailer, so a thread with a bare lens tag is left out. */
|
|
121
|
+
function outcomesOf(prs: PrRecord[]) {
|
|
122
|
+
const byRule = new Map<string, Record<Outcome, number>>();
|
|
123
|
+
|
|
124
|
+
for (const pr of prs) {
|
|
125
|
+
const fixed = new Set(pr.commits.filter((c) => !isReverted(c, pr.commits)).flatMap((c) => c.fixes).flatMap(rulesOf));
|
|
126
|
+
|
|
127
|
+
for (const t of pr.threads) {
|
|
128
|
+
const rules = rulesOf(t.tag);
|
|
129
|
+
const outcome: Outcome = t.humanReplied ? "argued" : !t.resolved ? "open" : rules.some((r) => fixed.has(r)) ? "fixed" : "dismissed";
|
|
130
|
+
|
|
131
|
+
for (const rule of rules) {
|
|
132
|
+
const entry = byRule.get(rule) ?? { fixed: 0, dismissed: 0, argued: 0, open: 0 };
|
|
133
|
+
entry[outcome] += 1;
|
|
134
|
+
byRule.set(rule, entry);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return [...byRule].map(([rule, counts]) => ({ rule, ...counts })).sort((a, b) => b.dismissed + b.argued - a.dismissed - a.argued || a.rule.localeCompare(b.rule));
|
|
108
140
|
}
|
|
109
141
|
|
|
110
142
|
/** Parse one shepherd thread's first comment and its replies; null when the thread is not shepherd's. */
|
|
@@ -129,12 +161,12 @@ export function parseCommit(sha: string, message: string, date = ""): Commit {
|
|
|
129
161
|
return { sha, headline: message.split("\n")[0] ?? "", skill: trailer("Shepherd"), lens: trailer("Shepherd-Lens"), fixes, date };
|
|
130
162
|
}
|
|
131
163
|
|
|
132
|
-
/** An incident whose culprit PR
|
|
164
|
+
/** An incident whose culprit PR gate approved: the review let a regression through. */
|
|
133
165
|
export type Escape = { incident: number; title: string; url: string; pr: number; sha: string; cause: "coverage_gap" | "merge_skew" | null };
|
|
134
166
|
|
|
135
|
-
const REGRESSED = /Regressed-by: #(\d+) \(([0-9a-f]{7,40})\),
|
|
167
|
+
const REGRESSED = /Regressed-by: #(\d+) \(([0-9a-f]{7,40})\), gate: approved\b/;
|
|
136
168
|
|
|
137
|
-
/** The escape an incident issue records, from the lines the incident skill writes; null unless
|
|
169
|
+
/** The escape an incident issue records, from the lines the incident skill writes; null unless gate approved the culprit. */
|
|
138
170
|
export function parseEscape(issue: { number: number; title: string; url: string; body: string }): Escape | null {
|
|
139
171
|
if (!issue.body.includes("<!-- shepherd-incident:")) return null;
|
|
140
172
|
const m = REGRESSED.exec(issue.body);
|
|
@@ -175,11 +207,11 @@ const Commits = z.array(z.object({ sha: z.string(), commit: z.object({ message:
|
|
|
175
207
|
|
|
176
208
|
const Statuses = z.object({ statuses: z.array(z.object({ context: z.string(), state: z.string(), description: z.string().nullable(), updated_at: z.string() })) });
|
|
177
209
|
|
|
178
|
-
const Comments = z.array(z.object({ body: z.string() }));
|
|
210
|
+
const Comments = z.array(z.object({ body: z.string(), user: z.object({ login: z.string(), type: z.string() }).nullable() }));
|
|
179
211
|
|
|
180
212
|
const Issues = z.array(z.object({ number: z.number(), title: z.string(), url: z.string(), body: z.string() }));
|
|
181
213
|
|
|
182
|
-
/** Incidents updated since `since` whose culprit PR
|
|
214
|
+
/** Incidents updated since `since` whose culprit PR gate approved. */
|
|
183
215
|
export function collectEscapes(repo: string, since: string): Escape[] {
|
|
184
216
|
const issues = Issues.parse(JSON.parse(gh(["issue", "list", "-R", repo, "--label", "incident", "--state", "all", "--search", `updated:>=${since}`, "--limit", "100", "--json", "number,title,url,body"])));
|
|
185
217
|
|
|
@@ -208,16 +240,19 @@ export function collect(repo: string, since: string): PrRecord[] {
|
|
|
208
240
|
return canPush.get(login) === true;
|
|
209
241
|
};
|
|
210
242
|
|
|
243
|
+
// Anyone can paste the header on a public PR, so shepherd's marks count only from a bot or an account that can push.
|
|
244
|
+
const canPost = (login: string | undefined, bot: boolean) => bot || (login !== undefined && trusted(login));
|
|
245
|
+
|
|
211
246
|
return prs.map((pr) => {
|
|
212
247
|
const query = "query($owner:String!,$name:String!,$n:Int!){repository(owner:$owner,name:$name){pullRequest(number:$n){reviewThreads(first:100){nodes{isResolved path comments(first:50){nodes{author{login __typename} body}}}}}}}";
|
|
213
248
|
|
|
214
249
|
const threads = Threads.parse(JSON.parse(gh(["api", "graphql", "-f", `query=${query}`, "-f", `owner=${owner}`, "-f", `name=${name}`, "-F", `n=${pr.number}`])))
|
|
215
|
-
.data.repository.pullRequest.reviewThreads.nodes.flatMap((t) => parseThread(t.path, t.isResolved, t.comments.nodes.map((c) => ({ author: c.author?.login ?? "ghost", bot: c.author?.__typename === "Bot", body: c.body }))) ?? []);
|
|
250
|
+
.data.repository.pullRequest.reviewThreads.nodes.filter((t) => canPost(t.comments.nodes[0]?.author?.login, t.comments.nodes[0]?.author?.__typename === "Bot")).flatMap((t) => parseThread(t.path, t.isResolved, t.comments.nodes.map((c) => ({ author: c.author?.login ?? "ghost", bot: c.author?.__typename === "Bot", body: c.body }))) ?? []);
|
|
216
251
|
|
|
217
252
|
const commits = Commits.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/pulls/${pr.number}/commits`])).flat()).map((c) => parseCommit(c.sha, c.commit.message, c.commit.committer?.date ?? ""));
|
|
218
253
|
|
|
219
254
|
const status = Statuses.parse(JSON.parse(gh(["api", `repos/${repo}/commits/${pr.headRefOid}/status`]))).statuses.find((s) => s.context === "shepherd");
|
|
220
|
-
const summary = Comments.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/issues/${pr.number}/comments`])).flat()).find((c) => c.body.includes("<!-- shepherd-swarm-summary -->"));
|
|
255
|
+
const summary = Comments.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/issues/${pr.number}/comments`])).flat()).find((c) => c.body.includes("<!-- shepherd-swarm-summary -->") && canPost(c.user?.login, c.user?.type === "Bot"));
|
|
221
256
|
|
|
222
257
|
return { number: pr.number, headSha: pr.headRefOid, title: pr.title, state: pr.state, threads, commits, status: status ? { state: status.state, description: status.description ?? "", updatedAt: status.updated_at } : null, rounds: roundsOf(summary?.body), trusted: trusted(pr.author.login) };
|
|
223
258
|
});
|
package/src/lens.ts
CHANGED
|
@@ -69,16 +69,17 @@ export function ciOpencodeConfig(session: string, choice: { ladder: string[] | n
|
|
|
69
69
|
};
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
-
//
|
|
72
|
+
// gate's `provider:model` names where OpenCode's provider id differs; every other provider keeps its name.
|
|
73
73
|
const OPENCODE_PROVIDER = new Map([["bedrock", "amazon-bedrock"], ["vertex", "google-vertex"]]);
|
|
74
74
|
|
|
75
75
|
/**
|
|
76
|
-
* A model id as OpenCode spells it.
|
|
76
|
+
* A model id as OpenCode spells it. gate's canonical `provider:model` (`bedrock:eu.anthropic.claude-…-v1:0`)
|
|
77
77
|
* becomes `provider/model` (`amazon-bedrock/eu.anthropic.claude-…-v1:0`), so one ladder serves both tools.
|
|
78
78
|
* An OpenCode id or a Claude Code alias passes through.
|
|
79
79
|
*/
|
|
80
80
|
export function opencodeModel(id: string): string {
|
|
81
|
-
|
|
81
|
+
// `provider:model` or `provider/model`, split at the first separator, so `bedrock/…` and `bedrock:…` both reach OpenCode as `amazon-bedrock/…`
|
|
82
|
+
const m = /^([a-z][a-z0-9-]*)[:/](.+)$/.exec(id);
|
|
82
83
|
|
|
83
84
|
return m ? `${OPENCODE_PROVIDER.get(m[1]!) ?? m[1]}/${m[2]}` : id;
|
|
84
85
|
}
|
|
@@ -106,8 +107,7 @@ export function setTier(file: string, tier: string, model: string): void {
|
|
|
106
107
|
write(file, { ...current, tiers: { ...current.tiers, [tier]: ModelId.parse(model) } });
|
|
107
108
|
}
|
|
108
109
|
|
|
109
|
-
// The environment variable each OpenCode provider reads its key from.
|
|
110
|
-
// model's provider needs, so a single-provider team sets one secret.
|
|
110
|
+
// The environment variable each OpenCode provider reads its key from.
|
|
111
111
|
const PROVIDER_KEY_ENV = new Map([
|
|
112
112
|
["opencode", "OPENCODE_API_KEY"],
|
|
113
113
|
["opencode-go", "OPENCODE_API_KEY"],
|
|
@@ -124,8 +124,20 @@ const PROVIDER_KEY_ENV = new Map([
|
|
|
124
124
|
|
|
125
125
|
export const PROVIDER_KEY_ENVS = [...PROVIDER_KEY_ENV.values()];
|
|
126
126
|
|
|
127
|
+
// Providers both halves can run: the loop through OpenCode, the gate through the AI SDK. google/ and xai/ are loop-only.
|
|
128
|
+
const SHARED_PROVIDERS = new Set(["anthropic", "openai", "opencode", "opencode-go", "openrouter", "deepseek", "groq", "mistral", "amazon-bedrock"]);
|
|
129
|
+
|
|
130
|
+
/** Why one model id cannot drive the loop and the gate, or null when it can. A bare name (a tier or an alias) passes. */
|
|
131
|
+
export function sharedModelProblem(id: string): string | null {
|
|
132
|
+
const provider = /[:/]/.test(id) ? opencodeModel(id).split("/")[0]! : null;
|
|
133
|
+
|
|
134
|
+
if (provider === null || SHARED_PROVIDERS.has(provider)) return null;
|
|
135
|
+
|
|
136
|
+
return `${id}: the loop and the gate both need to run it, and ${provider}/ is not one both can. Use ${[...SHARED_PROVIDERS].map((p) => `${p}/`).join(", ")}; or pin the loop with \`shepherd models pin loop\` and set \`model:\` in .shepherd/gate.yml separately.`;
|
|
137
|
+
}
|
|
138
|
+
|
|
127
139
|
/** The key variable for an OpenCode model id's provider, or null when shepherd does not know it. */
|
|
128
|
-
export const providerKeyEnv = (
|
|
140
|
+
export const providerKeyEnv = (id: string): string | null => PROVIDER_KEY_ENV.get(opencodeModel(id).split("/")[0]!) ?? null;
|
|
129
141
|
|
|
130
142
|
/** Replace the ladder, cheapest model first. */
|
|
131
143
|
export function setLadder(file: string, ladder: string[]): void {
|
|
@@ -172,7 +184,7 @@ fixes by its own rules. Nothing here can loosen shepherd's rules.
|
|
|
172
184
|
- Escalate: <changes that go to the author instead of being fixed>.
|
|
173
185
|
`;
|
|
174
186
|
|
|
175
|
-
// `- **<id>**: <rule>`, or `- **<id>** [refuse|escalate|note]: <rule>` to set that rule's severity for
|
|
187
|
+
// `- **<id>**: <rule>`, or `- **<id>** [refuse|escalate|note]: <rule>` to set that rule's severity for gate.
|
|
176
188
|
const RULE = /^- \*\*([a-z0-9][a-z0-9-]*)\*\*(?: \[(refuse|escalate|note)\])?:/gm;
|
|
177
189
|
|
|
178
190
|
export type LensRule = { id: string; severity?: Severity };
|
package/src/setup.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// First-run setup on GitHub: what a hosted console does for other tools, done here through `gh`, so nothing outside
|
|
2
|
+
// the repository and the user's own credentials is involved.
|
|
3
|
+
import { providerKeyEnv, sharedModelProblem } from "./lens.ts";
|
|
4
|
+
|
|
5
|
+
/** The model `shepherd init` picks from the first key it finds, in this order. */
|
|
6
|
+
export const KEY_MODELS = [
|
|
7
|
+
["OPENCODE_API_KEY", "opencode-go/deepseek-v4.1-flash"],
|
|
8
|
+
["ANTHROPIC_API_KEY", "anthropic/claude-opus-5"],
|
|
9
|
+
["OPENAI_API_KEY", "openai/gpt-5.5"],
|
|
10
|
+
["OPENROUTER_API_KEY", "openrouter/moonshotai/kimi-k3"],
|
|
11
|
+
] as const;
|
|
12
|
+
|
|
13
|
+
export type Pick = { model: string; key: string | null; why: string; chosen: boolean };
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The model to set up. An explicit choice (the flag) wins; then the model the policy already names, so a rerun never
|
|
17
|
+
* moves it; then SHEPHERD_MODEL; then the first provider key present. `chosen` is true only when it should be written.
|
|
18
|
+
*/
|
|
19
|
+
export function pickModel(flag: string | undefined, existing: { value: string; source: string } | undefined, env: Record<string, string | undefined>): Pick | null {
|
|
20
|
+
const at = (model: string, why: string, chosen: boolean): Pick => ({ model, key: providerKeyEnv(model), why, chosen });
|
|
21
|
+
|
|
22
|
+
if (flag) return at(flag, "--model", true);
|
|
23
|
+
|
|
24
|
+
if (existing) return at(existing.value, `models.loop from ${existing.source}`, false);
|
|
25
|
+
const named = env.SHEPHERD_MODEL?.trim();
|
|
26
|
+
|
|
27
|
+
if (named) return at(named, "SHEPHERD_MODEL", true);
|
|
28
|
+
const found = KEY_MODELS.find(([key]) => env[key]?.trim());
|
|
29
|
+
|
|
30
|
+
return found ? { model: found[1], key: found[0], why: `${found[0]} is set`, chosen: true } : null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Why the picked model cannot drive both halves, or null. */
|
|
34
|
+
export const modelProblem = (model: string) => sharedModelProblem(model);
|
|
35
|
+
|
|
36
|
+
export type Gh = (args: string[], input?: string) => { ok: boolean; out: string };
|
|
37
|
+
|
|
38
|
+
export type Secret = { name: string; value: string | undefined };
|
|
39
|
+
|
|
40
|
+
export type GithubSetup = { repo: string; secrets: Secret[] };
|
|
41
|
+
|
|
42
|
+
/** What setup did on GitHub: one line per step, and what is left for a person. */
|
|
43
|
+
export type SetupOutcome = { lines: string[]; todo: string[] };
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The repository-side steps, each safe to repeat and none fatal: each secret set from the environment (sent on
|
|
47
|
+
* stdin, never in arguments), and Actions allowed to approve PRs (the gate's approval counts only then). Committing
|
|
48
|
+
* the setup files stays with the user, as any other change does.
|
|
49
|
+
*/
|
|
50
|
+
export function githubSetup(gh: Gh, s: GithubSetup): SetupOutcome {
|
|
51
|
+
const lines: string[] = [];
|
|
52
|
+
const todo: string[] = [];
|
|
53
|
+
|
|
54
|
+
for (const { name, value } of s.secrets) {
|
|
55
|
+
if (value && gh(["secret", "set", name, "--repo", s.repo], value).ok) lines.push(`secret ${name} set on ${s.repo}`);
|
|
56
|
+
else {
|
|
57
|
+
lines.push(`secret ${name} ${value ? "could not be set" : "is not in your environment"}`);
|
|
58
|
+
todo.push(`gh secret set ${name} --repo ${s.repo}`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const current = gh(["api", `repos/${s.repo}/actions/permissions/workflow`, "--jq", ".default_workflow_permissions"]);
|
|
63
|
+
const allow = current.ok ? gh(["api", "-X", "PUT", `repos/${s.repo}/actions/permissions/workflow`, "-f", `default_workflow_permissions=${current.out.trim() || "read"}`, "-F", "can_approve_pull_request_reviews=true"]) : current;
|
|
64
|
+
|
|
65
|
+
if (allow.ok) lines.push("setting Actions may approve pull requests");
|
|
66
|
+
else {
|
|
67
|
+
lines.push("setting needs a repository admin");
|
|
68
|
+
todo.push('a repository admin: Settings → Actions → General → "Allow GitHub Actions to create and approve pull requests"');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return { lines, todo };
|
|
72
|
+
}
|
package/src/stack.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Stacked PRs and wrong bases, detected the way
|
|
1
|
+
// Stacked PRs and wrong bases, detected the way gate detects them before it reviews. A PR's base...head
|
|
2
2
|
// diff is what lands when it merges, which is right for a lone PR and wrong for two common shapes: a PR stacked on another open PR into the same base (its diff
|
|
3
3
|
// carries the parent's change), and a PR branched from one long-lived branch but opened against another
|
|
4
4
|
// (its diff carries everything the two branches differ by). Everything here is git ancestry on immutable
|
|
@@ -67,7 +67,7 @@ export function stackedParent(pr: { number: number; headSha: string; baseSha: st
|
|
|
67
67
|
return best && { parent: best.parent, parentHead: best.parentHead };
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
// The same default
|
|
70
|
+
// The same default gate's policy carries: branches a PR is opened from and merged into, never feature branches.
|
|
71
71
|
export const LONG_LIVED = ["main", "master", "develop", "release", "release/*", "staging"];
|
|
72
72
|
|
|
73
73
|
const View = z.object({ number: z.number(), headRefName: z.string(), headRefOid: z.string(), baseRefName: z.string() });
|
package/src/status.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// One answer to "where is my PR?": the gate's verdict, shepherd's status, people's reviews and CI on the PR's current
|
|
2
|
+
// head, folded into a single stage and the one thing to do next. Read-only: `shepherd status <pr>` gathers the facts.
|
|
3
|
+
|
|
4
|
+
export type Stage = "reviewing" | "fixing" | "waiting for CI" | "needs you" | "ready" | "merged" | "closed";
|
|
5
|
+
|
|
6
|
+
export type Run = { name: string; status: string; conclusion: string; url: string };
|
|
7
|
+
|
|
8
|
+
export type PrFacts = {
|
|
9
|
+
pr: number;
|
|
10
|
+
state: string; // OPEN, MERGED, CLOSED
|
|
11
|
+
head: string;
|
|
12
|
+
url: string;
|
|
13
|
+
/**
|
|
14
|
+
* The gate's latest verdict and the head it reviewed, from its own markers only (DISMISSED: an approval taken back).
|
|
15
|
+
* `kept`: an approval of an earlier head whose PR diff matches the current one byte for byte, which the gate keeps.
|
|
16
|
+
*/
|
|
17
|
+
gate?: { verdict: string; head: string; kept?: boolean };
|
|
18
|
+
/** The `shepherd` commit status on the head, if the loop has run on it. */
|
|
19
|
+
shepherd?: { state: string; description: string };
|
|
20
|
+
/** People whose latest review requests changes. */
|
|
21
|
+
changesRequested: string[];
|
|
22
|
+
/** Unresolved review threads, whoever opened them. */
|
|
23
|
+
openThreads: number;
|
|
24
|
+
/** Checks on the head: Actions workflow runs, other apps' check runs and commit statuses (CI outside Actions). */
|
|
25
|
+
runs: Run[];
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type PrStatus = { stage: Stage; lines: string[]; next: string };
|
|
29
|
+
|
|
30
|
+
const short = (sha: string) => sha.slice(0, 7);
|
|
31
|
+
|
|
32
|
+
export function prStatus(f: PrFacts): PrStatus {
|
|
33
|
+
const running = f.runs.filter((r) => r.status !== "completed");
|
|
34
|
+
// The gate's run fails on a refusal: its verdict, read from its review, speaks for it.
|
|
35
|
+
const failed = f.runs.filter((r) => r.status === "completed" && /^(failure|timed_out|startup_failure)$/.test(r.conclusion) && !/gate/.test(r.name));
|
|
36
|
+
const gateOnHead = f.gate && (f.gate.kept || f.head.startsWith(f.gate.head)) ? f.gate : undefined;
|
|
37
|
+
|
|
38
|
+
const lines = [
|
|
39
|
+
`PR #${f.pr} on ${short(f.head)} ${f.url}`,
|
|
40
|
+
`gate ${gateOnHead ? `${gateOnHead.verdict}${gateOnHead.kept ? ` (kept from ${short(gateOnHead.head)}: the PR's diff is unchanged)` : ""}` : f.gate ? `${f.gate.verdict} on ${short(f.gate.head)}, an older head` : "has not reviewed it"}`,
|
|
41
|
+
`shepherd ${f.shepherd ? `${f.shepherd.state}: ${f.shepherd.description}` : "has not run on this head"}`,
|
|
42
|
+
`CI ${f.runs.length ? f.runs.map((r) => `${r.name} ${r.status === "completed" ? r.conclusion : r.status}`).join(", ") : "no runs on this head"}`,
|
|
43
|
+
...(f.changesRequested.length ? [`reviews changes requested by ${f.changesRequested.join(", ")}`] : []),
|
|
44
|
+
...(f.openThreads ? [`threads ${f.openThreads} unresolved`] : []),
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
const at = (stage: Stage, next: string): PrStatus => ({ stage, lines, next });
|
|
48
|
+
|
|
49
|
+
if (f.state === "MERGED") return at("merged", "nothing: it is merged");
|
|
50
|
+
|
|
51
|
+
if (f.state === "CLOSED") return at("closed", "nothing: it is closed");
|
|
52
|
+
|
|
53
|
+
// What is running now comes first: a pending status only says shepherd is not done, not what it waits on.
|
|
54
|
+
if (running.some((r) => /gate/.test(r.name))) return at("reviewing", "wait: the gate is reviewing");
|
|
55
|
+
|
|
56
|
+
if (running.length) return at("waiting for CI", `wait: ${running.map((r) => r.name).join(", ")} running`);
|
|
57
|
+
|
|
58
|
+
// Then every current blocker, so an older success never reads as ready over a newer failure.
|
|
59
|
+
if (failed.length) return at("needs you", `fix ${failed.map((r) => r.name).join(", ")}, or comment /shepherd to have shepherd try`);
|
|
60
|
+
|
|
61
|
+
if (f.changesRequested.length) return at("needs you", `answer the review from ${f.changesRequested.join(", ")}`);
|
|
62
|
+
|
|
63
|
+
if (gateOnHead?.verdict === "ERROR") return at("needs you", "the gate could not review: open its run, then comment /gate");
|
|
64
|
+
|
|
65
|
+
if (gateOnHead?.verdict === "DISMISSED") return at("needs you", "the gate's approval was dismissed: comment /gate to review again");
|
|
66
|
+
|
|
67
|
+
if (gateOnHead && gateOnHead.verdict !== "APPROVED") return at("needs you", "read the gate's review, then push a fix or comment /shepherd");
|
|
68
|
+
|
|
69
|
+
if (f.shepherd?.state === "pending") return at("fixing", `wait: shepherd is working (${f.shepherd.description})`);
|
|
70
|
+
|
|
71
|
+
if (f.shepherd?.state === "failure") return at("needs you", `${f.shepherd.description}; then comment /shepherd`);
|
|
72
|
+
|
|
73
|
+
if (f.shepherd?.state === "error") return at("needs you", "shepherd stopped on an error: open its run in Actions, then comment /shepherd");
|
|
74
|
+
|
|
75
|
+
if (f.openThreads) return at("needs you", `resolve the ${f.openThreads} open review thread${f.openThreads === 1 ? "" : "s"}, or comment /shepherd to have shepherd work through them`);
|
|
76
|
+
|
|
77
|
+
if (!gateOnHead) return at("reviewing", f.gate ? "wait for the gate to review the new head, or comment /gate" : "comment /shepherd to start, or /gate for a review only");
|
|
78
|
+
|
|
79
|
+
return at("ready", "merge it");
|
|
80
|
+
}
|