@jagreehal/shepherd 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -0
- package/README.md +40 -419
- package/lenses/observability/SKILL.md +18 -11
- package/package.json +16 -8
- package/skills/ci-repair/SKILL.md +6 -1
- package/skills/garden/SKILL.md +59 -9
- package/skills/garden/references/telemetry.md +50 -0
- package/skills/incident/SKILL.md +20 -6
- package/skills/shepherd/SKILL.md +95 -1
- package/skills/shepherd/references/dispatch.md +12 -0
- package/skills/shepherd/references/stories.md +102 -0
- package/skills/swarm/SKILL.md +5 -0
- package/skills/triage/SKILL.md +19 -0
- package/src/cli.ts +118 -7
- package/src/garden-traces.ts +461 -0
- package/src/garden.ts +27 -1
- package/src/install.ts +3 -0
- package/src/notify.ts +138 -0
- package/src/trace.ts +180 -0
- package/templates/shepherd-incident.yml +156 -0
- package/templates/shepherd.yml +399 -0
package/src/garden.ts
CHANGED
|
@@ -12,6 +12,7 @@ export type Commit = { sha: string; headline: string; skill: string | null; lens
|
|
|
12
12
|
|
|
13
13
|
export type PrRecord = {
|
|
14
14
|
number: number;
|
|
15
|
+
headSha?: string;
|
|
15
16
|
title: string;
|
|
16
17
|
state: string;
|
|
17
18
|
threads: Thread[];
|
|
@@ -128,6 +129,22 @@ export function parseCommit(sha: string, message: string, date = ""): Commit {
|
|
|
128
129
|
return { sha, headline: message.split("\n")[0] ?? "", skill: trailer("Shepherd"), lens: trailer("Shepherd-Lens"), fixes, date };
|
|
129
130
|
}
|
|
130
131
|
|
|
132
|
+
/** An incident whose culprit PR stamp approved: the review let a regression through. */
|
|
133
|
+
export type Escape = { incident: number; title: string; url: string; pr: number; sha: string; cause: "coverage_gap" | "merge_skew" | null };
|
|
134
|
+
|
|
135
|
+
const REGRESSED = /Regressed-by: #(\d+) \(([0-9a-f]{7,40})\), stamp: approved\b/;
|
|
136
|
+
|
|
137
|
+
/** The escape an incident issue records, from the lines the incident skill writes; null unless stamp approved the culprit. */
|
|
138
|
+
export function parseEscape(issue: { number: number; title: string; url: string; body: string }): Escape | null {
|
|
139
|
+
if (!issue.body.includes("<!-- shepherd-incident:")) return null;
|
|
140
|
+
const m = REGRESSED.exec(issue.body);
|
|
141
|
+
|
|
142
|
+
if (!m) return null;
|
|
143
|
+
const cause = /^`?Coverage gap: /m.test(issue.body) ? "coverage_gap" : /^`?Merge skew: /m.test(issue.body) ? "merge_skew" : null;
|
|
144
|
+
|
|
145
|
+
return { incident: issue.number, title: issue.title, url: issue.url, pr: Number(m[1]), sha: m[2]!, cause };
|
|
146
|
+
}
|
|
147
|
+
|
|
131
148
|
export const roundsOf = (summaryBody: string | undefined) => (summaryBody ? Number(ROUND.exec(summaryBody)?.[1] ?? 0) || null : null);
|
|
132
149
|
|
|
133
150
|
const gh = (args: string[]) => execFileSync("gh", args, { encoding: "utf8", maxBuffer: 64 << 20, stdio: ["ignore", "pipe", "pipe"] }); // stderr rides on the thrown error
|
|
@@ -160,6 +177,15 @@ const Statuses = z.object({ statuses: z.array(z.object({ context: z.string(), st
|
|
|
160
177
|
|
|
161
178
|
const Comments = z.array(z.object({ body: z.string() }));
|
|
162
179
|
|
|
180
|
+
const Issues = z.array(z.object({ number: z.number(), title: z.string(), url: z.string(), body: z.string() }));
|
|
181
|
+
|
|
182
|
+
/** Incidents updated since `since` whose culprit PR stamp approved. */
|
|
183
|
+
export function collectEscapes(repo: string, since: string): Escape[] {
|
|
184
|
+
const issues = Issues.parse(JSON.parse(gh(["issue", "list", "-R", repo, "--label", "incident", "--state", "all", "--search", `updated:>=${since}`, "--limit", "100", "--json", "number,title,url,body"])));
|
|
185
|
+
|
|
186
|
+
return issues.flatMap((i) => parseEscape(i) ?? []);
|
|
187
|
+
}
|
|
188
|
+
|
|
163
189
|
/** Read shepherd's record on the PRs updated since `since` (YYYY-MM-DD). */
|
|
164
190
|
export function collect(repo: string, since: string): PrRecord[] {
|
|
165
191
|
const [owner, name] = repo.split("/");
|
|
@@ -193,6 +219,6 @@ export function collect(repo: string, since: string): PrRecord[] {
|
|
|
193
219
|
const status = Statuses.parse(JSON.parse(gh(["api", `repos/${repo}/commits/${pr.headRefOid}/status`]))).statuses.find((s) => s.context === "shepherd");
|
|
194
220
|
const summary = Comments.parse(JSON.parse(gh(["api", "--paginate", "--slurp", `repos/${repo}/issues/${pr.number}/comments`])).flat()).find((c) => c.body.includes("<!-- shepherd-swarm-summary -->"));
|
|
195
221
|
|
|
196
|
-
return { number: pr.number, title: pr.title, state: pr.state, threads, commits, status: status ? { state: status.state, description: status.description ?? "", updatedAt: status.updated_at } : null, rounds: roundsOf(summary?.body), trusted: trusted(pr.author.login) };
|
|
222
|
+
return { number: pr.number, headSha: pr.headRefOid, title: pr.title, state: pr.state, threads, commits, status: status ? { state: status.state, description: status.description ?? "", updatedAt: status.updated_at } : null, rounds: roundsOf(summary?.body), trusted: trusted(pr.author.login) };
|
|
197
223
|
});
|
|
198
224
|
}
|
package/src/install.ts
CHANGED
|
@@ -6,6 +6,9 @@ import path from "node:path";
|
|
|
6
6
|
|
|
7
7
|
export const MARKER = ".shepherd-installed";
|
|
8
8
|
|
|
9
|
+
/** A workflow template with every `SHEPHERD_VERSION:` pin set to `version`, so each job runs the CLI that wrote it. */
|
|
10
|
+
export const pinWorkflow = (text: string, version: string) => text.replaceAll(/SHEPHERD_VERSION: \S+/g, `SHEPHERD_VERSION: ${version}`);
|
|
11
|
+
|
|
9
12
|
// Command files are single files, so they carry the marker as a comment line instead.
|
|
10
13
|
const COMMAND_MARKER = "<!-- shepherd-installed -->";
|
|
11
14
|
|
package/src/notify.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// Outbound notifications. Off until SHEPHERD_NOTIFY_WEBHOOK is set. A Slack Incoming Webhook URL gets a
|
|
2
|
+
// Slack message; any other URL gets the event as JSON, for Discord relays, Teams, n8n, Zapier or an agent
|
|
3
|
+
// platform's trigger. The caller runs `shepherd notify` right after setting the commit status; this
|
|
4
|
+
// module drops repeats (a local record across heads, a `shepherd/notify` status on the head for fresh CI
|
|
5
|
+
// runners), so a /loop or a rerun tells people once per change of state, not every iteration. People answer
|
|
6
|
+
// on the PR, where the next iteration's triage reads them.
|
|
7
|
+
import { execFileSync } from "node:child_process";
|
|
8
|
+
import { createHash } from "node:crypto";
|
|
9
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { z } from "zod";
|
|
12
|
+
import { stateDir } from "./local.ts";
|
|
13
|
+
|
|
14
|
+
export const STATES = ["success", "failure", "pending", "error"] as const;
|
|
15
|
+
|
|
16
|
+
export type State = (typeof STATES)[number];
|
|
17
|
+
|
|
18
|
+
export type NotifyEvent = {
|
|
19
|
+
event: "shepherd.state";
|
|
20
|
+
repo: string;
|
|
21
|
+
pr: number;
|
|
22
|
+
url: string;
|
|
23
|
+
head_sha: string;
|
|
24
|
+
state: State;
|
|
25
|
+
description: string;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** States that notify unless SHEPHERD_NOTIFY_ON says otherwise: done, needs a person, and stopped by an error. */
|
|
29
|
+
export const NOTIFY_DEFAULT: State[] = ["success", "failure", "error"];
|
|
30
|
+
|
|
31
|
+
export function notifyOn(raw: string | undefined): State[] {
|
|
32
|
+
const asked = new Set((raw ?? "").split(",").map((s) => s.trim().toLowerCase()));
|
|
33
|
+
const list = STATES.filter((s) => asked.has(s));
|
|
34
|
+
|
|
35
|
+
return list.length ? list : NOTIFY_DEFAULT;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const LABEL: Record<State, string> = { success: "ready to merge", failure: "needs you", pending: "working", error: "stopped" };
|
|
39
|
+
|
|
40
|
+
// Descriptions can quote PR titles, reviewers and bot output; Slack reads <...> as links and mentions (<!channel>).
|
|
41
|
+
export const slackEscape = (t: string) => t.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
|
42
|
+
|
|
43
|
+
export const isSlack = (url: string) => url.startsWith("https://hooks.slack.com/services/");
|
|
44
|
+
|
|
45
|
+
export function payload(url: string, e: NotifyEvent): string {
|
|
46
|
+
return JSON.stringify(isSlack(url) ? { text: `*shepherd ${LABEL[e.state]}* <${e.url}|${e.repo}#${e.pr}>\n${slackEscape(e.description)}` } : e);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export type Sent = "sent" | "unchanged" | "off" | "failed";
|
|
50
|
+
|
|
51
|
+
/** Where the last delivered (or deliberately skipped) state is kept. Written only after it is settled. */
|
|
52
|
+
export type Ledger = { read(): string | null; write(token: string): void };
|
|
53
|
+
|
|
54
|
+
const token = (e: NotifyEvent) => createHash("sha256").update(`${e.state}\n${e.description}`).digest("hex").slice(0, 16);
|
|
55
|
+
|
|
56
|
+
/** A file under the git directory: survives a local /loop across heads. */
|
|
57
|
+
export function fileLedger(repoRoot: string, pr: number): Ledger {
|
|
58
|
+
// Resolved on use: outside a repository, read and write throw and notify treats that as no record.
|
|
59
|
+
const file = () => path.join(stateDir(repoRoot), `notified-${pr}`);
|
|
60
|
+
|
|
61
|
+
return {
|
|
62
|
+
read: () => (existsSync(file()) ? readFileSync(file(), "utf8").trim() : null),
|
|
63
|
+
write: (t) => {
|
|
64
|
+
mkdirSync(path.dirname(file()), { recursive: true });
|
|
65
|
+
writeFileSync(file(), t);
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const Statuses = z.array(z.object({ context: z.string(), description: z.string().nullable() }));
|
|
71
|
+
|
|
72
|
+
export const LEDGER_CONTEXT = "shepherd/notify";
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* A `shepherd/notify` commit status on the head: survives fresh CI runners, and is separate from the `shepherd`
|
|
76
|
+
* status, so a status that was set but never delivered is retried. GitHub lists statuses newest first.
|
|
77
|
+
*/
|
|
78
|
+
export function statusLedger(repo: string, head: string, cwd: string): Ledger {
|
|
79
|
+
const gh = (args: string[]) => execFileSync("gh", args, { cwd, encoding: "utf8", timeout: 15_000 });
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
read: () => {
|
|
83
|
+
const latest = Statuses.parse(JSON.parse(gh(["api", `repos/${repo}/commits/${head}/statuses?per_page=100`]))).find((st) => st.context === LEDGER_CONTEXT);
|
|
84
|
+
|
|
85
|
+
return /([0-9a-f]{16})$/.exec(latest?.description ?? "")?.[1] ?? null;
|
|
86
|
+
},
|
|
87
|
+
write: (t) => {
|
|
88
|
+
gh(["api", `repos/${repo}/statuses/${head}`, "-f", `context=${LEDGER_CONTEXT}`, "-f", "state=success", "-f", `description=last notified ${t}`]);
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Posts the event when its state is wanted and differs from the last one settled for this PR. A state that is
|
|
95
|
+
* not wanted is settled without a post, so failure → pending → failure tells people twice. A failed post is
|
|
96
|
+
* not settled, so the next call retries it. A ledger that cannot be read or written never stops the call.
|
|
97
|
+
* Never throws.
|
|
98
|
+
*/
|
|
99
|
+
export async function notify(url: string | undefined, e: NotifyEvent, on: State[], ledgers: Ledger[], post: typeof fetch = fetch): Promise<Sent> {
|
|
100
|
+
if (!url) return "off";
|
|
101
|
+
const t = token(e);
|
|
102
|
+
|
|
103
|
+
for (const l of ledgers) {
|
|
104
|
+
try {
|
|
105
|
+
if (l.read() === t) return "unchanged";
|
|
106
|
+
} catch {
|
|
107
|
+
// unreadable: treat as not sent
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const settle = () => {
|
|
112
|
+
for (const l of ledgers) {
|
|
113
|
+
try {
|
|
114
|
+
l.write(t);
|
|
115
|
+
} catch {
|
|
116
|
+
// a repeat next time is better than failing this iteration
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
if (!on.includes(e.state)) {
|
|
122
|
+
settle();
|
|
123
|
+
|
|
124
|
+
return "off";
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
const res = await post(url, { method: "POST", headers: { "content-type": "application/json" }, body: payload(url, e), signal: AbortSignal.timeout(10_000) });
|
|
129
|
+
|
|
130
|
+
if (!res.ok) return "failed";
|
|
131
|
+
} catch {
|
|
132
|
+
return "failed";
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
settle();
|
|
136
|
+
|
|
137
|
+
return "sent";
|
|
138
|
+
}
|
package/src/trace.ts
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// Iteration traces. The agent drives shepherd's steps, so this CLI cannot watch them run. The skill marks each
|
|
2
|
+
// step's start and end with `shepherd trace event`, which takes the time from this machine, and
|
|
3
|
+
// `shepherd trace flush` turns the record into one `shepherd.iteration` span with a child span per step.
|
|
4
|
+
// Spans export through autotel only when OTEL_EXPORTER_OTLP_ENDPOINT is set. Model usage inside a step is the
|
|
5
|
+
// harness's to report: these spans time the steps and record their outcomes, nothing inside them.
|
|
6
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
import { stateDir } from "./local.ts";
|
|
10
|
+
|
|
11
|
+
// `stamp` is the wait for stamp's verdict.
|
|
12
|
+
export const STAGES = ["checks", "swarm", "verify", "triage", "simplify", "ci-repair", "stamp"] as const;
|
|
13
|
+
|
|
14
|
+
const Event = z.object({
|
|
15
|
+
at: z.string(),
|
|
16
|
+
stage: z.enum(STAGES),
|
|
17
|
+
phase: z.enum(["start", "end"]),
|
|
18
|
+
round: z.number().int().positive().optional(),
|
|
19
|
+
outcome: z.enum(["ok", "failed", "skipped"]).optional(),
|
|
20
|
+
retries: z.number().int().nonnegative().optional(),
|
|
21
|
+
head_sha: z.string().regex(/^[0-9a-f]{7,40}$/).optional(),
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
export type TraceEvent = z.infer<typeof Event>;
|
|
25
|
+
|
|
26
|
+
export type Stage = {
|
|
27
|
+
stage: TraceEvent["stage"];
|
|
28
|
+
round: number | undefined;
|
|
29
|
+
start: string;
|
|
30
|
+
end: string;
|
|
31
|
+
outcome: "ok" | "failed" | "skipped" | "unfinished";
|
|
32
|
+
retries: number;
|
|
33
|
+
head_sha: string | undefined;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/** The record for one PR's current iteration, inside the git directory like shepherd's other state. */
|
|
37
|
+
export const traceFile = (repoRoot: string, pr: number) => path.join(stateDir(repoRoot), `trace-${pr}.jsonl`);
|
|
38
|
+
|
|
39
|
+
/** A step event as the CLI receives it, before validation. */
|
|
40
|
+
export type EventInput = { stage: string; phase: string; round?: number | undefined; outcome?: string | undefined; retries?: number | undefined; head_sha?: string | undefined };
|
|
41
|
+
|
|
42
|
+
/** Append one step event, stamped now. Throws on an unknown stage or a malformed field. */
|
|
43
|
+
export function recordEvent(file: string, event: EventInput, now = new Date()): TraceEvent {
|
|
44
|
+
const parsed = Event.safeParse({ ...event, at: now.toISOString() });
|
|
45
|
+
|
|
46
|
+
if (!parsed.success) throw new Error(`trace event: ${parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`);
|
|
47
|
+
|
|
48
|
+
const e = parsed.data;
|
|
49
|
+
|
|
50
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
51
|
+
appendFileSync(file, JSON.stringify(e) + "\n");
|
|
52
|
+
|
|
53
|
+
return e;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const parseLine = (line: string): TraceEvent[] => {
|
|
57
|
+
try {
|
|
58
|
+
const parsed = Event.safeParse(JSON.parse(line));
|
|
59
|
+
|
|
60
|
+
return parsed.success ? [parsed.data] : [];
|
|
61
|
+
} catch {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** The steps of one iteration, and when its first event happened. */
|
|
67
|
+
export type StageRecord = { start: string | null; stages: Stage[] };
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Steps from the record: each start pairs with the next end of the same stage and round, or of the same stage
|
|
71
|
+
* when the end's round differs from every open start's. A start with no end closes at `now` as unfinished; an
|
|
72
|
+
* end with no start, or a line that does not parse, is dropped.
|
|
73
|
+
*/
|
|
74
|
+
export function stagesFrom(text: string, now: Date): StageRecord {
|
|
75
|
+
const events = text.split("\n").filter((l) => l.trim()).flatMap(parseLine);
|
|
76
|
+
const open = new Map<string, TraceEvent>();
|
|
77
|
+
const stages: Stage[] = [];
|
|
78
|
+
|
|
79
|
+
for (const e of events) {
|
|
80
|
+
const exact = `${e.stage}#${e.round ?? 0}`;
|
|
81
|
+
const key = e.phase === "end" && !open.has(exact) ? ([...open.keys()].filter((k) => k.startsWith(`${e.stage}#`)).at(-1) ?? exact) : exact;
|
|
82
|
+
const started = open.get(key);
|
|
83
|
+
|
|
84
|
+
if (e.phase === "start") open.set(key, e);
|
|
85
|
+
else if (started) {
|
|
86
|
+
open.delete(key);
|
|
87
|
+
stages.push({ stage: e.stage, round: started.round ?? e.round, start: started.at, end: e.at, outcome: e.outcome ?? "ok", retries: e.retries ?? 0, head_sha: e.head_sha ?? started.head_sha });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
for (const s of open.values()) stages.push({ stage: s.stage, round: s.round, start: s.at, end: now.toISOString(), outcome: "unfinished", retries: s.retries ?? 0, head_sha: s.head_sha });
|
|
92
|
+
|
|
93
|
+
return { start: events[0]?.at ?? null, stages: stages.sort((a, b) => a.start.localeCompare(b.start)) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** What ties an iteration to the PR, the CI run and other traces: stamp's review, a CI run's trace. */
|
|
97
|
+
export type Iteration = {
|
|
98
|
+
repo: string;
|
|
99
|
+
pr: number;
|
|
100
|
+
headSha: string;
|
|
101
|
+
baseSha: string | undefined;
|
|
102
|
+
outcome: string;
|
|
103
|
+
ciRun: number | undefined;
|
|
104
|
+
ciAttempt: number | undefined;
|
|
105
|
+
// W3C traceparent values; each becomes a span link, so separate processes join without one long trace.
|
|
106
|
+
links: string[];
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const TRACEPARENT = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/;
|
|
110
|
+
|
|
111
|
+
/** The iteration span's attributes, named like stamp's so one query finds both. An absent value stays undefined. */
|
|
112
|
+
export const iterationAttributes = (it: Iteration, stages: Stage[]) => ({
|
|
113
|
+
"vcs.repository.name": it.repo,
|
|
114
|
+
"vcs.change.id": String(it.pr),
|
|
115
|
+
"vcs.ref.head.revision": it.headSha,
|
|
116
|
+
"vcs.ref.base.revision": it.baseSha,
|
|
117
|
+
"cicd.pipeline.run.id": it.ciRun === undefined ? undefined : String(it.ciRun),
|
|
118
|
+
"shepherd.ci.attempt": it.ciAttempt,
|
|
119
|
+
"shepherd.outcome": it.outcome,
|
|
120
|
+
"shepherd.rounds": Math.max(0, ...stages.map((s) => s.round ?? 0)),
|
|
121
|
+
"shepherd.retries": stages.reduce((n, s) => n + s.retries, 0),
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
export const stageAttributes = (s: Stage) => ({
|
|
125
|
+
"shepherd.stage": s.stage,
|
|
126
|
+
"shepherd.outcome": s.outcome,
|
|
127
|
+
"shepherd.retries": s.retries,
|
|
128
|
+
"shepherd.round": s.round,
|
|
129
|
+
"vcs.ref.head.revision": s.head_sha,
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
// OpenTelemetry rejects an undefined attribute value, so absent ones are left off.
|
|
133
|
+
const defined = (attrs: ReturnType<typeof iterationAttributes> | ReturnType<typeof stageAttributes>) => Object.fromEntries(Object.entries(attrs).filter(([, v]) => v !== undefined));
|
|
134
|
+
|
|
135
|
+
/** Export the iteration and its steps through autotel. The SDK loads only here, so an unset endpoint costs nothing. */
|
|
136
|
+
async function exportSpans(start: string, it: Iteration, stages: Stage[], now: Date): Promise<void> {
|
|
137
|
+
const { init, shutdown, getTracer, runWithSpan, SpanStatusCode } = await import("autotel");
|
|
138
|
+
|
|
139
|
+
init({ service: process.env.OTEL_SERVICE_NAME || "shepherd" });
|
|
140
|
+
|
|
141
|
+
const tracer = getTracer("shepherd");
|
|
142
|
+
|
|
143
|
+
const links = it.links.flatMap((l) => {
|
|
144
|
+
const m = TRACEPARENT.exec(l);
|
|
145
|
+
|
|
146
|
+
return m ? [{ context: { traceId: m[1]!, spanId: m[2]!, traceFlags: parseInt(m[3]!, 16) } }] : [];
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
const root = tracer.startSpan("shepherd.iteration", { startTime: new Date(start), links, attributes: defined(iterationAttributes(it, stages)) });
|
|
150
|
+
|
|
151
|
+
for (const s of stages) {
|
|
152
|
+
const span = runWithSpan(root, () => tracer.startSpan(`shepherd.${s.stage}`, { startTime: new Date(s.start), attributes: defined(stageAttributes(s)) }));
|
|
153
|
+
|
|
154
|
+
if (s.outcome === "failed" || s.outcome === "unfinished") span.setStatus({ code: SpanStatusCode.ERROR });
|
|
155
|
+
span.end(new Date(s.end));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
root.end(now);
|
|
159
|
+
await shutdown();
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Close the iteration: read its record, export it when an endpoint is set, and clear the record for the next one.
|
|
164
|
+
* The record clears whether or not the export reaches the collector, and the result carries the export's error.
|
|
165
|
+
*/
|
|
166
|
+
export async function flush(file: string, it: Iteration, now = new Date()): Promise<{ stages: Stage[]; exported: boolean; error?: string }> {
|
|
167
|
+
const { start, stages } = stagesFrom(existsSync(file) ? readFileSync(file, "utf8") : "", now);
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT || !start) return { stages, exported: false };
|
|
171
|
+
|
|
172
|
+
await exportSpans(start, it, stages, now);
|
|
173
|
+
|
|
174
|
+
return { stages, exported: true };
|
|
175
|
+
} catch (e) {
|
|
176
|
+
return { stages, exported: false, error: e instanceof Error ? e.message : String(e) };
|
|
177
|
+
} finally {
|
|
178
|
+
rmSync(file, { force: true });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# shepherd incident in CI: a red run on main, or an error sent in by hand or by an alerting tool, becomes an
|
|
2
|
+
# incident issue, a reproducing test and a fix PR. The PR then goes through CI, stamp and the shepherd
|
|
3
|
+
# workflow like any other. Installed by `shepherd workflows`.
|
|
4
|
+
#
|
|
5
|
+
# Triggers:
|
|
6
|
+
# CI fails on a push to the default branch -> the run URL is the signal
|
|
7
|
+
# workflow_dispatch with `error` -> paste an error or stack trace
|
|
8
|
+
# repository_dispatch type `incident`, payload.error -> an alerting tool (Sentry, Grafana, PagerDuty, a script):
|
|
9
|
+
# gh api repos/<owner>/<repo>/dispatches -f event_type=incident -F 'client_payload[error]=<error text>'
|
|
10
|
+
# Edit `workflows: [ci]` to name your CI workflow. workflow_run runs this file from the default branch only.
|
|
11
|
+
#
|
|
12
|
+
# The signal is untrusted text (alert payloads carry user input), so the agent's job can read the repo and
|
|
13
|
+
# file issues, nothing more: no contents write, no credential in its checkout. It hands its commit over as a
|
|
14
|
+
# git bundle; the `open` job, which runs no model, pushes only an `incident/<n>` branch that builds on the
|
|
15
|
+
# default branch and changes nothing outside INCIDENT_PATHS (no workflows, manifests, lockfiles or config:
|
|
16
|
+
# CI on the PR installs and runs them), then opens the PR. The agent still holds the model key.
|
|
17
|
+
#
|
|
18
|
+
# Secrets: SHEPHERD_TOKEN (contents, pull requests: write) for `open`, so the fix PR starts CI and stamp;
|
|
19
|
+
# with github.token the PR opens but no workflow runs on it. OPENCODE_API_KEY for the model.
|
|
20
|
+
# Variables: SHEPHERD_MODEL (provider/model), INCIDENT_PATHS (extended regex; empty = ^src/).
|
|
21
|
+
name: shepherd-incident
|
|
22
|
+
on:
|
|
23
|
+
workflow_run:
|
|
24
|
+
workflows: [ci]
|
|
25
|
+
types: [completed]
|
|
26
|
+
workflow_dispatch:
|
|
27
|
+
inputs:
|
|
28
|
+
error:
|
|
29
|
+
description: Error, stack trace or alert text
|
|
30
|
+
required: true
|
|
31
|
+
repository_dispatch:
|
|
32
|
+
types: [incident]
|
|
33
|
+
|
|
34
|
+
permissions: {}
|
|
35
|
+
|
|
36
|
+
jobs:
|
|
37
|
+
investigate:
|
|
38
|
+
if: >-
|
|
39
|
+
(github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'failure'
|
|
40
|
+
&& github.event.workflow_run.event == 'push'
|
|
41
|
+
&& github.event.workflow_run.head_branch == github.event.repository.default_branch)
|
|
42
|
+
|| github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
timeout-minutes: 45
|
|
45
|
+
# On the job, not the workflow: every green CI run starts this workflow and skips, and a workflow-level group
|
|
46
|
+
# would hand that skip the one pending slot a real incident needs. One at a time, so a repeat signal dedupes.
|
|
47
|
+
concurrency:
|
|
48
|
+
group: shepherd-incident
|
|
49
|
+
cancel-in-progress: false
|
|
50
|
+
permissions:
|
|
51
|
+
contents: read
|
|
52
|
+
issues: write
|
|
53
|
+
actions: read
|
|
54
|
+
env:
|
|
55
|
+
SHEPHERD_VERSION: 0.3.0 # pinned: this runs with the repo's secrets, so bumps are deliberate
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/checkout@v4
|
|
58
|
+
with:
|
|
59
|
+
ref: ${{ github.event.repository.default_branch }}
|
|
60
|
+
fetch-depth: 0
|
|
61
|
+
persist-credentials: false
|
|
62
|
+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0: pinned, this job holds the model key
|
|
63
|
+
# Install your project's dependencies here so the reproducing test can run, with install scripts off and no
|
|
64
|
+
# dependency cache (npm ci --ignore-scripts; setup-node without `cache:`): this job runs the model.
|
|
65
|
+
- shell: bash # -eo pipefail: a failed download fails here, not three steps later
|
|
66
|
+
run: |
|
|
67
|
+
# The pinned CLI on PATH: the skills call `shepherd`, and fall back to an unpinned bunx without it.
|
|
68
|
+
bun add -g "@jagreehal/shepherd@$SHEPHERD_VERSION"
|
|
69
|
+
export PATH="$HOME/.bun/bin:$PATH" && echo "$HOME/.bun/bin" >> "$GITHUB_PATH"
|
|
70
|
+
[ "$(command -v shepherd)" = "$HOME/.bun/bin/shepherd" ] || { echo "shepherd is not the pinned CLI"; exit 1; }
|
|
71
|
+
shepherd install
|
|
72
|
+
curl -fsSL https://opencode.ai/install | bash -s -- --version 1.19.0 # 1.x: this installer refuses 2.x
|
|
73
|
+
"$HOME/.opencode/bin/opencode" --version # fails the step if the install did not land
|
|
74
|
+
echo "$HOME/.opencode/bin" >> "$GITHUB_PATH"
|
|
75
|
+
git config user.name "github-actions[bot]"
|
|
76
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
77
|
+
mkdir -p "$RUNNER_TEMP/handoff"
|
|
78
|
+
- name: Write the signal
|
|
79
|
+
env: # through env, never ${{ }} in the script: the error text is untrusted
|
|
80
|
+
RUN_URL: ${{ github.event.workflow_run.html_url }}
|
|
81
|
+
INPUT_ERROR: ${{ github.event.inputs.error }}
|
|
82
|
+
PAYLOAD_ERROR: ${{ github.event.client_payload.error }}
|
|
83
|
+
run: |
|
|
84
|
+
if [ -n "$RUN_URL" ]; then echo "$RUN_URL" > "$RUNNER_TEMP/signal.txt"
|
|
85
|
+
else printf '%s\n' "${INPUT_ERROR:-$PAYLOAD_ERROR}" > "$RUNNER_TEMP/signal.txt"; fi
|
|
86
|
+
- name: Incident
|
|
87
|
+
timeout-minutes: 20 # opencode retries a provider error (an exhausted quota) silently; fail instead of holding the queue
|
|
88
|
+
env:
|
|
89
|
+
GH_TOKEN: ${{ github.token }}
|
|
90
|
+
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
|
|
91
|
+
OPENCODE_DISABLE_PROJECT_CONFIG: "1"
|
|
92
|
+
OPENCODE_DISABLE_AUTOUPDATE: "1"
|
|
93
|
+
MODEL: ${{ vars.SHEPHERD_MODEL || 'opencode-go/kimi-k3' }}
|
|
94
|
+
HANDOFF: ${{ runner.temp }}/handoff
|
|
95
|
+
run: |
|
|
96
|
+
opencode run --auto -m "$MODEL" --title "incident" "Load the incident skill and run it on the signal in $RUNNER_TEMP/signal.txt
|
|
97
|
+
(a run URL, or an error or alert text).
|
|
98
|
+
You are running in GitHub Actions on the default branch, already checked out, with no push access.
|
|
99
|
+
At Step 6 do not push or open the PR: commit on incident/<issue>, write the PR title to $HANDOFF/pr-title.txt
|
|
100
|
+
and the body to $HANDOFF/pr-body.md; the next job pushes and opens it.
|
|
101
|
+
Nobody is watching this session: never wait for an answer. Anything you would ask is reported in the final JSON."
|
|
102
|
+
- name: Bundle the fix branch
|
|
103
|
+
env:
|
|
104
|
+
DEFAULT: ${{ github.event.repository.default_branch }}
|
|
105
|
+
run: |
|
|
106
|
+
branch=$(git branch --list 'incident/*' --format='%(refname:short)' | head -1)
|
|
107
|
+
[ -n "$branch" ] || { echo "no fix branch: see the incident issue"; exit 0; }
|
|
108
|
+
echo "$branch" > "$RUNNER_TEMP/handoff/branch"
|
|
109
|
+
git bundle create "$RUNNER_TEMP/handoff/fix.bundle" "origin/$DEFAULT..$branch"
|
|
110
|
+
- uses: actions/upload-artifact@v4
|
|
111
|
+
with:
|
|
112
|
+
name: incident-handoff
|
|
113
|
+
path: ${{ runner.temp }}/handoff
|
|
114
|
+
if-no-files-found: ignore
|
|
115
|
+
|
|
116
|
+
open:
|
|
117
|
+
needs: investigate
|
|
118
|
+
runs-on: ubuntu-latest
|
|
119
|
+
env:
|
|
120
|
+
INCIDENT_PATHS: ${{ vars.INCIDENT_PATHS || '^src/' }} # extended regex every changed path must match
|
|
121
|
+
permissions:
|
|
122
|
+
contents: write
|
|
123
|
+
pull-requests: write
|
|
124
|
+
steps:
|
|
125
|
+
- uses: actions/download-artifact@v4
|
|
126
|
+
id: handoff
|
|
127
|
+
continue-on-error: true # no artifact: no fix branch
|
|
128
|
+
with:
|
|
129
|
+
name: incident-handoff
|
|
130
|
+
path: handoff
|
|
131
|
+
- if: steps.handoff.outcome == 'success' && hashFiles('handoff/fix.bundle') != ''
|
|
132
|
+
uses: actions/checkout@v4
|
|
133
|
+
with:
|
|
134
|
+
ref: ${{ github.event.repository.default_branch }}
|
|
135
|
+
fetch-depth: 0
|
|
136
|
+
path: repo
|
|
137
|
+
token: ${{ secrets.SHEPHERD_TOKEN || github.token }}
|
|
138
|
+
- if: steps.handoff.outcome == 'success' && hashFiles('handoff/fix.bundle') != ''
|
|
139
|
+
working-directory: repo
|
|
140
|
+
shell: bash # -eo pipefail
|
|
141
|
+
env:
|
|
142
|
+
GH_TOKEN: ${{ secrets.SHEPHERD_TOKEN || github.token }}
|
|
143
|
+
DEFAULT: ${{ github.event.repository.default_branch }}
|
|
144
|
+
run: |
|
|
145
|
+
branch=$(cat ../handoff/branch)
|
|
146
|
+
[[ "$branch" =~ ^incident/[0-9]+$ ]] || { echo "::error::unexpected branch name"; exit 1; }
|
|
147
|
+
git fetch -q ../handoff/fix.bundle "$branch:$branch"
|
|
148
|
+
git merge-base "origin/$DEFAULT" "$branch" >/dev/null || { echo "::error::$branch shares no history with $DEFAULT"; exit 1; } # behind is fine: ci-repair updates it
|
|
149
|
+
# NUL-delimited and rename-free: a quoted or renamed path cannot slip past the match. The diff runs on
|
|
150
|
+
# its own so a failure stops here instead of passing an empty list; an empty change is refused too.
|
|
151
|
+
git diff -z --no-renames --name-only "origin/$DEFAULT...$branch" > ../changed
|
|
152
|
+
[ -s ../changed ] || { echo "::error::$branch changes nothing"; exit 1; }
|
|
153
|
+
outside=$(tr '\0' '\n' < ../changed | grep -Ev -- "$INCIDENT_PATHS" || [ $? -eq 1 ])
|
|
154
|
+
[ -z "$outside" ] || { echo "::error::the fix changes paths outside $INCIDENT_PATHS; a person should look:"; echo "$outside"; exit 1; }
|
|
155
|
+
git push origin "$branch:refs/heads/$branch"
|
|
156
|
+
gh pr create --base "$DEFAULT" --head "$branch" --title "$(head -c 200 ../handoff/pr-title.txt)" --body-file ../handoff/pr-body.md
|