@jagreehal/shepherd 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -5
- package/README.md +40 -27
- package/THIRD_PARTY_NOTICES.md +2 -0
- package/gate/.shepherd/gate.schema.json +647 -0
- package/gate/.shepherd/gate.yml +122 -0
- package/gate/.shepherd/review-guidance.md +95 -0
- package/gate/src/actions.ts +32 -0
- package/gate/src/cli.ts +1028 -0
- package/gate/src/config.ts +102 -0
- package/gate/src/delta.ts +29 -0
- package/gate/src/env.ts +30 -0
- package/gate/src/eval.ts +288 -0
- package/gate/src/familiarity.ts +461 -0
- package/gate/src/github.ts +761 -0
- package/gate/src/inline.ts +59 -0
- package/gate/src/isolated-cli.ts +281 -0
- package/gate/src/leads.ts +257 -0
- package/gate/src/llm.ts +606 -0
- package/gate/src/notify.ts +57 -0
- package/gate/src/policy.ts +1297 -0
- package/gate/src/provider-errors.ts +80 -0
- package/gate/src/retention.ts +86 -0
- package/gate/src/reviewer.ts +739 -0
- package/gate/src/signals.ts +105 -0
- package/gate/src/smoke-live.ts +64 -0
- package/gate/src/stack.ts +87 -0
- package/gate/src/stories.ts +297 -0
- package/gate/templates/policy-strict.yml +127 -0
- package/gate/templates/reviewer.Dockerfile +8 -0
- package/gate/templates/rules/observability/SKILL.md +33 -0
- package/gate/templates/shepherd-gate-actions.yml +35 -0
- package/gate/templates/shepherd-gate-edited.yml +29 -0
- package/gate/templates/shepherd-gate.yml +111 -0
- package/lenses/dotnet/SKILL.md +1 -1
- package/lenses/go/SKILL.md +1 -1
- package/lenses/java/SKILL.md +1 -1
- package/lenses/nextjs/SKILL.md +1 -1
- package/lenses/node-api/SKILL.md +1 -1
- package/lenses/nx-monorepo/SKILL.md +1 -1
- package/lenses/observability/SKILL.md +1 -1
- package/lenses/owasp-asvs/SKILL.md +1 -1
- package/lenses/owasp-llm/SKILL.md +1 -1
- package/lenses/pstack/SKILL.md +1 -1
- package/lenses/python/SKILL.md +1 -1
- package/lenses/react/SKILL.md +1 -1
- package/lenses/rust/SKILL.md +1 -1
- package/package.json +20 -6
- package/schema/config.json +1 -1
- package/skills/branch-review/SKILL.md +123 -0
- package/skills/ci-repair/SKILL.md +79 -35
- package/skills/garden/SKILL.md +29 -10
- package/skills/garden/references/telemetry.md +4 -4
- package/skills/incident/SKILL.md +19 -9
- package/skills/lens-import/SKILL.md +7 -7
- package/skills/review-maintainability/SKILL.md +1 -1
- package/skills/review-slop/SKILL.md +1 -1
- package/skills/shepherd/SKILL.md +59 -44
- package/skills/shepherd/references/degradation.md +3 -3
- package/skills/shepherd/references/dispatch.md +2 -2
- package/skills/shepherd/references/formats.md +5 -5
- package/skills/shepherd/references/models.md +2 -2
- package/skills/shepherd/references/policy.md +9 -9
- package/skills/swarm/SKILL.md +26 -17
- package/skills/swarm/references/custom-lenses.md +2 -2
- package/skills/swarm/references/formats.md +1 -1
- package/skills/triage/SKILL.md +11 -11
- package/src/bin.ts +10 -0
- package/src/cli.ts +283 -41
- package/src/config.ts +26 -25
- package/src/doctor.ts +31 -18
- package/src/garden-traces.ts +13 -13
- package/src/garden.ts +43 -8
- package/src/lens.ts +19 -7
- package/src/setup.ts +72 -0
- package/src/stack.ts +2 -2
- package/src/status.ts +80 -0
- package/src/summary.ts +52 -9
- package/src/trace.ts +4 -4
- package/src/working.ts +30 -0
- package/templates/shepherd-audit.yml +7 -15
- package/templates/shepherd-incident.yml +9 -12
- package/templates/shepherd.yml +39 -40
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// Risk signals from Jev, TypeSafe's System One model: calibrated probabilities for bounded questions
|
|
2
|
+
// about the diff, no prose. It runs before the reviewer and answers the question the gates cannot
|
|
3
|
+
// (does this change *behaviorally* enter risky territory?) and the one the reviewer would otherwise
|
|
4
|
+
// self-report. Advisory only: the probabilities are shown to the reviewer, decide whether a second
|
|
5
|
+
// opinion is worth its cost, and go in the evidence bundle for later calibration. Never a gate, and
|
|
6
|
+
// nothing here can loosen one. Skipped when TYPESAFE_API_KEY is unset.
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import type { PR } from "./github.ts";
|
|
9
|
+
|
|
10
|
+
export const SIGNAL_THRESHOLD = 0.7; // a probability at or above this is a flag; tune from the evidence bundles
|
|
11
|
+
|
|
12
|
+
const DIFF_CAP = 150_000;
|
|
13
|
+
|
|
14
|
+
// One yes/no question per dimension, each independently useful. The risky-territory set mirrors the
|
|
15
|
+
// review guidance; the last three are the agent failure modes the guidance tells the reviewer to read first.
|
|
16
|
+
export const QUESTIONS = {
|
|
17
|
+
auth: { instructions: "Does the change alter authentication, authorization, session, permission, or credential handling?", true: "It changes who can do what, how identity is established, or how secrets are handled.", false: "It does not touch those flows, even if the word appears." },
|
|
18
|
+
billing: { instructions: "Does the change alter billing, payment, quota, plan, or pricing logic?", true: "Money, entitlements, or limits behave differently.", false: "No billing or plan behavior changes." },
|
|
19
|
+
data_model: { instructions: "Does the change alter a database schema, a migration, a stored data format, or a public API contract?", true: "Persisted shapes, migrations, or an externally consumed contract change.", false: "Internal code only; stored data and public contracts are unchanged." },
|
|
20
|
+
ci_build: { instructions: "Does the change alter CI, build, deploy, or release tooling and configuration?", true: "How the project is built, tested in CI, deployed, or released changes.", false: "Application code or docs only." },
|
|
21
|
+
dependencies: { instructions: "Does the change add, remove, or upgrade a third-party dependency, or change install-time or lifecycle scripts?", true: "New or changed third-party code, or scripts that run on install or in CI.", false: "Dependencies and scripts are unchanged." },
|
|
22
|
+
write_path: { instructions: "Does the change alter a path that writes, deletes, or ingests user or customer data?", true: "Data is written, deleted, transformed, or ingested differently.", false: "Read-only, presentational, or unrelated to data writes." },
|
|
23
|
+
prompt_from_user_input: { instructions: "Does the change feed user-controlled text into a language model prompt or tool call?", true: "Untrusted text reaches an LLM call or an agent's instructions.", false: "No LLM prompt is built from user input." },
|
|
24
|
+
weakens_tests: { instructions: "Does the change weaken tests: delete tests, skip them, or rewrite assertions so they pass without verifying the original behavior?", true: "Coverage or assertion strength went down.", false: "Tests were added, tightened, or left alone; refactors that keep the assertions equivalent count as unchanged." },
|
|
25
|
+
weakens_ci: { instructions: "Does the change loosen a lint, type-check, coverage, or hook configuration?", true: "A rule was disabled or downgraded, an ignore widened, a threshold lowered, a strict flag turned off, or a hook removed.", false: "Configuration was tightened or not touched." },
|
|
26
|
+
undisclosed_behavior: { instructions: "Does the diff contain substantive behavior that the title and description do not mention?", true: "The description omits a real behavioral change present in the diff.", false: "The description covers what the diff does." },
|
|
27
|
+
} as const;
|
|
28
|
+
|
|
29
|
+
export type Signal = keyof typeof QUESTIONS;
|
|
30
|
+
|
|
31
|
+
export type Signals = Record<Signal, number>;
|
|
32
|
+
|
|
33
|
+
export const SIGNAL_IDS = ["auth", "billing", "data_model", "ci_build", "dependencies", "write_path", "prompt_from_user_input", "weakens_tests", "weakens_ci", "undisclosed_behavior"] as const satisfies readonly Signal[];
|
|
34
|
+
|
|
35
|
+
type Noul = { type: "noul"; instructions: string; criteria: { true: string; false: string } };
|
|
36
|
+
|
|
37
|
+
export type SystemOneRequest = {
|
|
38
|
+
model: "jev-latest";
|
|
39
|
+
state: { title: string; description: string; files: string[]; diff: string };
|
|
40
|
+
questions: Record<Signal, Noul>;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/** What the reviewer needs from the pull request; nothing else is sent. */
|
|
44
|
+
export type SignalInput = Pick<PR, "title" | "body" | "files" | "diff">;
|
|
45
|
+
|
|
46
|
+
const Response = z.object({
|
|
47
|
+
answers: z.record(z.string(), z.object({ type: z.literal("noul"), noul: z.number().min(0).max(1) })),
|
|
48
|
+
usage: z.object({ input_tokens: z.number().optional(), output_tokens: z.number().optional() }).optional(),
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
export function requestBody(pr: SignalInput): SystemOneRequest {
|
|
52
|
+
const questions: Partial<Record<Signal, Noul>> = {};
|
|
53
|
+
|
|
54
|
+
for (const id of SIGNAL_IDS) questions[id] = { type: "noul", instructions: QUESTIONS[id].instructions, criteria: { true: QUESTIONS[id].true, false: QUESTIONS[id].false } };
|
|
55
|
+
|
|
56
|
+
return {
|
|
57
|
+
model: "jev-latest",
|
|
58
|
+
state: {
|
|
59
|
+
title: pr.title,
|
|
60
|
+
description: pr.body,
|
|
61
|
+
files: pr.files.map((f) => `${f.filename} (+${f.additions}/-${f.deletions})`),
|
|
62
|
+
diff: pr.diff.slice(0, DIFF_CAP),
|
|
63
|
+
},
|
|
64
|
+
// SAFETY: the loop above assigned every id in SIGNAL_IDS, which enumerates every Signal.
|
|
65
|
+
questions: questions as Record<Signal, Noul>,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export async function riskSignals(pr: SignalInput): Promise<Signals | null> {
|
|
70
|
+
const key = process.env.TYPESAFE_API_KEY;
|
|
71
|
+
|
|
72
|
+
if (!key) return null;
|
|
73
|
+
|
|
74
|
+
const res = await fetch("https://api.typesafe.ai/v1/systemone", {
|
|
75
|
+
method: "POST",
|
|
76
|
+
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
|
|
77
|
+
body: JSON.stringify(requestBody(pr)),
|
|
78
|
+
signal: AbortSignal.timeout(30_000),
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
if (!res.ok) throw new Error(`typesafe ${res.status}: ${(await res.text()).slice(0, 300)}`);
|
|
82
|
+
const parsed = Response.parse(await res.json());
|
|
83
|
+
const out: Partial<Signals> = {};
|
|
84
|
+
|
|
85
|
+
for (const id of SIGNAL_IDS) {
|
|
86
|
+
const a = parsed.answers[id];
|
|
87
|
+
|
|
88
|
+
if (!a) throw new Error(`typesafe: no answer for ${id}`);
|
|
89
|
+
out[id] = a.noul;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// SAFETY: the loop above assigned every id in SIGNAL_IDS or threw, so no signal is missing.
|
|
93
|
+
return out as Signals;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The signals at or above the threshold, strongest first. */
|
|
97
|
+
export const flagged = (s: Signals, threshold = SIGNAL_THRESHOLD): Signal[] =>
|
|
98
|
+
SIGNAL_IDS.filter((id) => s[id] >= threshold).sort((a, b) => s[b] - s[a]);
|
|
99
|
+
|
|
100
|
+
/** The trusted-context block for the reviewer prompt. */
|
|
101
|
+
export function formatSignals(s: Signals): string {
|
|
102
|
+
const line = (id: Signal) => ` ${id}: ${s[id].toFixed(2)}${s[id] >= SIGNAL_THRESHOLD ? " ← flag" : ""}`;
|
|
103
|
+
|
|
104
|
+
return ["Risk signals (Jev, calibrated probability that the answer is yes; advisory, never a gate):", ...SIGNAL_IDS.map(line)].join("\n");
|
|
105
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Live smoke: runs the real reviewer over a fake PR against this checkout.
|
|
2
|
+
// Verifies toolRunner + output_config.format parse to a verdict. Costs one small call.
|
|
3
|
+
// ANTHROPIC_API_KEY=... bun run src/smoke-live.ts
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { review, reviewers } from "./reviewer.ts";
|
|
7
|
+
import type { PR } from "./github.ts";
|
|
8
|
+
|
|
9
|
+
const root = path.resolve(import.meta.dir, "..");
|
|
10
|
+
|
|
11
|
+
const pr: PR = {
|
|
12
|
+
repo: "example/smoke",
|
|
13
|
+
number: 0,
|
|
14
|
+
title: "fix: treat an empty tiers map as T1d instead of throwing",
|
|
15
|
+
body: "## Why\ntier() should not depend on the map having entries.\n\n## What changed\n- comment only, no behaviour change\n\n## Evidence\n- tests: bun test → 9 pass",
|
|
16
|
+
author: "smoke",
|
|
17
|
+
authorAssociation: "OWNER",
|
|
18
|
+
isFork: false,
|
|
19
|
+
isDraft: false,
|
|
20
|
+
mergeable: "MERGEABLE",
|
|
21
|
+
baseRef: "main",
|
|
22
|
+
headRef: "feat",
|
|
23
|
+
defaultBranch: "main",
|
|
24
|
+
headSha: "0000000",
|
|
25
|
+
baseSha: "0000000",
|
|
26
|
+
labels: [],
|
|
27
|
+
files: [{ filename: "src/policy.ts", additions: 1, deletions: 0, status: "modified" }],
|
|
28
|
+
reviews: [],
|
|
29
|
+
inline: [],
|
|
30
|
+
discussion: [],
|
|
31
|
+
reactions: [],
|
|
32
|
+
commits: [],
|
|
33
|
+
diff: `--- a/src/policy.ts
|
|
34
|
+
+++ b/src/policy.ts
|
|
35
|
+
@@ -116,6 +116,7 @@ export function tier(policy: Policy, files: PRFile[], denied: string[]): { tier:
|
|
36
|
+
const { lines, files: n } = substantiveSize(files);
|
|
37
|
+
+ // An empty tiers map falls through to T1d-complex below.
|
|
38
|
+
for (const [name, t] of Object.entries(policy.tiers)) {
|
|
39
|
+
if (lines <= t.max_lines && n <= t.max_files) return { tier: "T1-agent", sub: name };
|
|
40
|
+
}`,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const verdict = await review(
|
|
44
|
+
reviewers()[0]!,
|
|
45
|
+
{
|
|
46
|
+
pr,
|
|
47
|
+
gates: [
|
|
48
|
+
{ gate: "prerequisites", passed: true, message: "ready" },
|
|
49
|
+
{ gate: "deny-list", passed: true, message: "no sensitive paths" },
|
|
50
|
+
{ gate: "size", passed: true, message: "1 substantive lines / 1 files (limit 800/30)" },
|
|
51
|
+
{ gate: "tier", passed: true, message: "T1-agent / T1a-trivial" },
|
|
52
|
+
],
|
|
53
|
+
gateVerdict: "PASSED",
|
|
54
|
+
tier: "T1-agent / T1a-trivial",
|
|
55
|
+
titleFlags: [],
|
|
56
|
+
manifests: [],
|
|
57
|
+
scrutiny: [],
|
|
58
|
+
},
|
|
59
|
+
readFileSync(path.join(root, ".shepherd/review-guidance.md"), "utf8"),
|
|
60
|
+
root,
|
|
61
|
+
true,
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
console.log(JSON.stringify(verdict, null, 2));
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// Stacked PRs and wrong bases. A PR's base...head diff is what lands when it merges, which is right for a
|
|
2
|
+
// lone PR and wrong for two common shapes: a PR stacked on another open PR into the same base (its diff
|
|
3
|
+
// carries the parent's change), and a PR branched from one long-lived branch but opened against another
|
|
4
|
+
// (its diff carries everything the two branches differ by). Everything here is git ancestry on immutable
|
|
5
|
+
// SHAs; nothing a PR author writes (body, title, "stacked on #n") is read.
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { isOurs, reviewedShas, type ReviewRecord } from "./github.ts";
|
|
8
|
+
import { TRUSTED_ASSOCIATIONS } from "./policy.ts";
|
|
9
|
+
|
|
10
|
+
export type StackDeps = {
|
|
11
|
+
/** `git rev-list --topo-order <head> ^<base>`: this PR's commits, newest first. */
|
|
12
|
+
commits: (head: string, base: string) => string[];
|
|
13
|
+
/** How many of this PR's commits are not on `branch` (a remote-tracking ref). */
|
|
14
|
+
notOn: (head: string, base: string, branch: string) => number;
|
|
15
|
+
/** Remote branch names, `git ls-remote --heads origin`. */
|
|
16
|
+
branches: () => string[];
|
|
17
|
+
/** Fetch these branches into their remote-tracking refs. */
|
|
18
|
+
fetch: (branches: string[]) => void;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type WrongBase = { branch: string; on: number; total: number };
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The long-lived branch this PR's commits mostly already sit on, when that is not its base: it was branched
|
|
25
|
+
* from there and opened against the wrong target. A branch holding every commit of the PR is not flagged
|
|
26
|
+
* (a backport), and a PR whose head is itself a long-lived branch is a promotion, so it never flags.
|
|
27
|
+
*/
|
|
28
|
+
export function wrongBase(pr: { headSha: string; baseSha: string; baseRef: string; headRef: string }, longLived: string[], deps: StackDeps): WrongBase | null {
|
|
29
|
+
if (!longLived.length || longLived.some((g) => path.matchesGlob(pr.headRef, g))) return null;
|
|
30
|
+
const candidates = deps.branches().filter((b) => b !== pr.baseRef && b !== pr.headRef && longLived.some((g) => path.matchesGlob(b, g)));
|
|
31
|
+
|
|
32
|
+
if (!candidates.length) return null;
|
|
33
|
+
const total = deps.commits(pr.headSha, pr.baseSha).length;
|
|
34
|
+
|
|
35
|
+
if (total < 2) return null;
|
|
36
|
+
deps.fetch(candidates);
|
|
37
|
+
let worst: WrongBase | null = null;
|
|
38
|
+
|
|
39
|
+
for (const branch of candidates) {
|
|
40
|
+
const off = deps.notOn(pr.headSha, pr.baseSha, branch);
|
|
41
|
+
const on = total - off;
|
|
42
|
+
|
|
43
|
+
if (off > 0 && on * 2 > total && (!worst || on > worst.on)) worst = { branch, on, total };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return worst;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The nearest open PR into the same base whose head is one of this PR's own commits: the parent this PR is
|
|
51
|
+
* stacked on. Its head being in base..head is the whole test, so a PR already merged into the base, or one
|
|
52
|
+
* that only shares history, never counts.
|
|
53
|
+
*/
|
|
54
|
+
export function stackedParent(pr: { number: number; headSha: string; baseSha: string }, open: { number: number; headRefOid: string }[], deps: Pick<StackDeps, "commits">): { parent: number; parentHead: string } | null {
|
|
55
|
+
const others = open.filter((p) => p.number !== pr.number && p.headRefOid !== pr.headSha);
|
|
56
|
+
|
|
57
|
+
if (!others.length) return null;
|
|
58
|
+
const order = deps.commits(pr.headSha, pr.baseSha);
|
|
59
|
+
let best: { parent: number; parentHead: string; at: number } | null = null;
|
|
60
|
+
|
|
61
|
+
for (const p of others) {
|
|
62
|
+
const at = order.indexOf(p.headRefOid);
|
|
63
|
+
|
|
64
|
+
if (at >= 0 && (!best || at < best.at)) best = { parent: p.number, parentHead: p.headRefOid, at };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return best && { parent: best.parent, parentHead: best.parentHead };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Whether a PR is approved on `head` by someone whose approval carries authority: a reviewer with write access
|
|
72
|
+
* (OWNER, MEMBER, COLLABORATOR) or gate itself. On a public repository anyone can submit an APPROVED review,
|
|
73
|
+
* so an outsider's never counts. A gate approval counts for the head its reviewed marker names, never its
|
|
74
|
+
* commit_id. Some trusted reviewer's latest decision approves exactly that commit, and no
|
|
75
|
+
* reviewer's latest decision, trusted or not, requests changes. A COMMENTED review is not a decision.
|
|
76
|
+
*/
|
|
77
|
+
export function approvedOn(reviews: ReviewRecord[], head: string, botLogin: string): boolean {
|
|
78
|
+
const latest = new Map<string, ReviewRecord>();
|
|
79
|
+
|
|
80
|
+
for (const r of reviews) if (r.state !== "COMMENTED" && r.state !== "PENDING") latest.set(r.user.login, r);
|
|
81
|
+
const decisions = [...latest.values()];
|
|
82
|
+
// The head an approval covers. GitHub moves commit_id forward when a branch is updated from its base, so a
|
|
83
|
+
// gate approval covers only the head its reviewed marker names, and one without a marker covers none.
|
|
84
|
+
const covers = (r: ReviewRecord) => (isOurs(r, botLogin) ? (reviewedShas(r)?.head ?? null) : TRUSTED_ASSOCIATIONS.has(r.author_association ?? "") ? r.commit_id : null);
|
|
85
|
+
|
|
86
|
+
return decisions.some((r) => r.state === "APPROVED" && covers(r) === head) && !decisions.some((r) => r.state === "CHANGES_REQUESTED");
|
|
87
|
+
}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
// Test evidence from the repository's own CI: the executable-stories review JSON its tests wrote for the
|
|
2
|
+
// commit under review. It reaches the reviewer inside the untrusted fence, and nothing in it moves a gate,
|
|
3
|
+
// grants an approval or counts as independent assurance: the PR's author wrote the tests that produced it.
|
|
4
|
+
// It can only hold an approval back: a `weakened` finding sends the PR to a person (weakenedCount).
|
|
5
|
+
// Matching the commit makes the report relevant to this head. It does not make the report trustworthy.
|
|
6
|
+
import { execFileSync } from "node:child_process";
|
|
7
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { tmpdir } from "node:os";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
|
|
12
|
+
/** `stories:` in policy.yml, read from the default branch like the rest of the policy. */
|
|
13
|
+
export const StoriesPolicySchema = z
|
|
14
|
+
.object({
|
|
15
|
+
// The workflow file whose runs upload the report, e.g. ci.yml.
|
|
16
|
+
workflow: z.string().min(1),
|
|
17
|
+
// The artifact name that workflow uploads.
|
|
18
|
+
artifact: z.string().min(1),
|
|
19
|
+
// The report's path inside the artifact. unzip reads it as a pattern, so wildcards are refused.
|
|
20
|
+
file: z
|
|
21
|
+
.string()
|
|
22
|
+
.refine((p) => !path.isAbsolute(p) && !p.split(/[\\/]/).includes("..") && !/[*?[\]\\]/.test(p), "a relative path inside the artifact, without wildcards")
|
|
23
|
+
.default("index.review.json"),
|
|
24
|
+
max_bytes: z.number().int().positive().max(10_000_000).default(1_000_000),
|
|
25
|
+
})
|
|
26
|
+
.strict();
|
|
27
|
+
|
|
28
|
+
export type StoriesPolicy = z.infer<typeof StoriesPolicySchema>;
|
|
29
|
+
|
|
30
|
+
const Strength = z.enum(["none", "weak", "moderate", "strong"]);
|
|
31
|
+
|
|
32
|
+
// The fields gate reads from the executable-stories review report, version 1.
|
|
33
|
+
const ReviewReport = z.object({
|
|
34
|
+
version: z.literal(1),
|
|
35
|
+
// Where the tests ran, from the raw run. The report's headRef is only the comparison it was asked for.
|
|
36
|
+
execution: z.object({ gitSha: z.string().optional(), runScope: z.enum(["full", "filtered"]).optional() }).optional(),
|
|
37
|
+
run: z.object({ total: z.number(), passed: z.number(), failed: z.number(), skipped: z.number(), pending: z.number() }),
|
|
38
|
+
findings: z.array(
|
|
39
|
+
z.object({
|
|
40
|
+
kind: z.enum(["failed", "unasserted", "weakened", "skipped", "uncovered", "weak", "policy"]),
|
|
41
|
+
severity: z.enum(["blocker", "major", "minor"]),
|
|
42
|
+
title: z.string(),
|
|
43
|
+
file: z.string().optional(),
|
|
44
|
+
line: z.number().optional(),
|
|
45
|
+
detail: z.string(),
|
|
46
|
+
}),
|
|
47
|
+
),
|
|
48
|
+
changedFiles: z.array(z.object({ path: z.string(), band: z.enum(["uncovered", "weak", "covered"]), claims: z.array(z.object({ scenario: z.string(), strength: Strength })) })),
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
export type ReviewReport = z.infer<typeof ReviewReport>;
|
|
52
|
+
|
|
53
|
+
export type StoryRun = { id: number; url: string; attempt: number; event: string; status: string; conclusion: string; headSha: string; createdAt: string };
|
|
54
|
+
|
|
55
|
+
export type StoryEvidence =
|
|
56
|
+
| { status: "missing"; reason: string }
|
|
57
|
+
| { status: "pending"; run: StoryRun }
|
|
58
|
+
| { status: "invalid"; run: StoryRun; reason: string }
|
|
59
|
+
| { status: "stale"; run: StoryRun; tested: string }
|
|
60
|
+
| { status: "ok"; run: StoryRun; tested: "head" | "merge"; scope: "full" | "filtered" | undefined; report: ReviewReport };
|
|
61
|
+
|
|
62
|
+
export type StoryDeps = {
|
|
63
|
+
/** Runs of the workflow for one commit. */
|
|
64
|
+
runs: (workflow: string, sha: string) => StoryRun[];
|
|
65
|
+
/** One file's text from a run's artifact, at most maxBytes; null when the run has no artifact by that name. Throws when the file cannot be read within bounds. */
|
|
66
|
+
report: (runId: number, artifact: string, file: string, maxBytes: number) => string | null;
|
|
67
|
+
/** A commit's parent SHAs. */
|
|
68
|
+
parents: (sha: string) => string[];
|
|
69
|
+
/** Whether `sha` is `from` or a descendant of it. */
|
|
70
|
+
descends: (sha: string, from: string) => boolean;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const GhRuns = z.array(
|
|
74
|
+
z.object({ databaseId: z.number(), url: z.string(), attempt: z.number(), event: z.string(), status: z.string(), conclusion: z.string(), headSha: z.string(), createdAt: z.string() }),
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
const GhArtifacts = z.object({ artifacts: z.array(z.object({ id: z.number(), name: z.string(), size_in_bytes: z.number(), expired: z.boolean() })) });
|
|
78
|
+
|
|
79
|
+
export const ARCHIVE_LIMIT = 50_000_000;
|
|
80
|
+
|
|
81
|
+
// execFileSync's error when the output passed maxBuffer.
|
|
82
|
+
const Overflow = z.object({ code: z.literal("ENOBUFS") });
|
|
83
|
+
|
|
84
|
+
/** One entry of a zip, read without extracting anything to disk. The entry is cut off past maxBytes, so a compressed bomb never inflates. */
|
|
85
|
+
export function readEntry(zip: string, file: string, maxBytes: number, timeout: number): string {
|
|
86
|
+
if (!execFileSync("unzip", ["-Z1", zip], { encoding: "utf8", timeout, maxBuffer: 10_000_000 }).split("\n").includes(file)) throw new Error(`${file} is not in the artifact`);
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
return execFileSync("unzip", ["-p", zip, file], { encoding: "utf8", timeout, maxBuffer: maxBytes });
|
|
90
|
+
} catch (e) {
|
|
91
|
+
if (Overflow.safeParse(e).success) throw new Error(`${file} is over the ${maxBytes}-byte limit`);
|
|
92
|
+
|
|
93
|
+
throw e;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** StoryDeps through gh, against one repository. Each call is bounded by the caller's timeout. */
|
|
98
|
+
export function ghStoryDeps(repo: string, cwd: string, timeout: number): StoryDeps {
|
|
99
|
+
const gh = (args: string[]) => execFileSync("gh", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout });
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
runs: (workflow, sha) =>
|
|
103
|
+
GhRuns.parse(JSON.parse(gh(["run", "list", "-R", repo, "--workflow", workflow, "--commit", sha, "--limit", "20", "--json", "databaseId,url,attempt,event,status,conclusion,headSha,createdAt"]))).map(
|
|
104
|
+
({ databaseId, ...r }) => ({ id: databaseId, ...r }),
|
|
105
|
+
),
|
|
106
|
+
report: (runId, artifact, file, maxBytes) => {
|
|
107
|
+
const found = GhArtifacts.parse(JSON.parse(gh(["api", `repos/${repo}/actions/runs/${runId}/artifacts?name=${encodeURIComponent(artifact)}`]))).artifacts.find(
|
|
108
|
+
(a) => a.name === artifact && !a.expired,
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
if (!found) return null;
|
|
112
|
+
|
|
113
|
+
if (found.size_in_bytes > ARCHIVE_LIMIT) throw new Error(`the artifact is ${found.size_in_bytes} bytes zipped, over the ${ARCHIVE_LIMIT}-byte limit`);
|
|
114
|
+
|
|
115
|
+
const dir = mkdtempSync(path.join(tmpdir(), "gate-stories-"));
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const zip = path.join(dir, "artifact.zip");
|
|
119
|
+
|
|
120
|
+
try {
|
|
121
|
+
// GitHub's size is its own report; maxBuffer is the bound that holds.
|
|
122
|
+
writeFileSync(zip, execFileSync("gh", ["api", `repos/${repo}/actions/artifacts/${found.id}/zip`], { cwd, stdio: ["ignore", "pipe", "pipe"], timeout, maxBuffer: ARCHIVE_LIMIT }));
|
|
123
|
+
} catch (e) {
|
|
124
|
+
if (Overflow.safeParse(e).success) throw new Error(`the artifact is over the ${ARCHIVE_LIMIT}-byte limit`);
|
|
125
|
+
|
|
126
|
+
throw e;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return readEntry(zip, file, maxBytes, timeout);
|
|
130
|
+
} finally {
|
|
131
|
+
rmSync(dir, { recursive: true, force: true });
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
parents: (sha) => z.array(z.string()).parse(JSON.parse(gh(["api", `repos/${repo}/commits/${sha}`, "--jq", "[.parents[].sha]"]))),
|
|
135
|
+
descends: (sha, from) => ["ahead", "identical"].includes(gh(["api", `repos/${repo}/compare/${from}...${sha}`, "--jq", ".status"]).trim()),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** The run that tested this head: the newest attempt of the newest run whose head is exactly this commit. */
|
|
140
|
+
export const latestRun = (runs: StoryRun[], headSha: string) =>
|
|
141
|
+
runs.filter((r) => r.headSha === headSha).sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.attempt - a.attempt)[0];
|
|
142
|
+
|
|
143
|
+
/** The report's text, validated as executable-stories' review report. */
|
|
144
|
+
export function parseReport(text: string, file: string): { ok: true; report: ReviewReport } | { ok: false; reason: string } {
|
|
145
|
+
let json: unknown;
|
|
146
|
+
|
|
147
|
+
try {
|
|
148
|
+
json = JSON.parse(text);
|
|
149
|
+
} catch {
|
|
150
|
+
return { ok: false, reason: `${file} is not JSON` };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const parsed = ReviewReport.safeParse(json);
|
|
154
|
+
|
|
155
|
+
return parsed.success ? { ok: true, report: parsed.data } : { ok: false, reason: `${file} is not an executable-stories review report v1: ${parsed.error.issues[0]?.message ?? "invalid"}` };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Which commit the report's tests ran on, from the SHA its run recorded. A pull_request workflow checks out
|
|
160
|
+
* GitHub's synthetic merge of the head into the base: that counts only when its parents are this head and
|
|
161
|
+
* the reviewed base or a commit descended from it. Null when the report records no SHA.
|
|
162
|
+
*/
|
|
163
|
+
export function testedCommit(report: ReviewReport, headSha: string, baseSha: string, { parents, descends }: Pick<StoryDeps, "parents" | "descends">): "head" | "merge" | string | null {
|
|
164
|
+
const tested = report.execution?.gitSha;
|
|
165
|
+
|
|
166
|
+
if (!tested) return null;
|
|
167
|
+
|
|
168
|
+
if (tested === headSha) return "head";
|
|
169
|
+
|
|
170
|
+
// The report names this, so only a full SHA reaches the reviewer's trusted context.
|
|
171
|
+
if (!/^[0-9a-f]{40}$/.test(tested)) return "an unrecognised ref";
|
|
172
|
+
|
|
173
|
+
try {
|
|
174
|
+
const [base, head, ...rest] = parents(tested);
|
|
175
|
+
|
|
176
|
+
// GitHub merges into the base branch as it is when the run starts, which can be past the base SHA the
|
|
177
|
+
// PR reports. Newer base code still ran this head's tests; a merge into an older base did not.
|
|
178
|
+
return head === headSha && rest.length === 0 && base !== undefined && (base === baseSha || descends(base, baseSha)) ? "merge" : tested;
|
|
179
|
+
} catch {
|
|
180
|
+
return tested;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export function loadStoryEvidence(policy: StoriesPolicy, headSha: string, baseSha: string, deps: StoryDeps): StoryEvidence {
|
|
185
|
+
let run: StoryRun | undefined;
|
|
186
|
+
|
|
187
|
+
try {
|
|
188
|
+
run = latestRun(deps.runs(policy.workflow, headSha), headSha);
|
|
189
|
+
} catch (e) {
|
|
190
|
+
return { status: "missing", reason: `cannot list ${policy.workflow} runs: ${e instanceof Error ? e.message.split("\n")[0] : e}` };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
if (!run) return { status: "missing", reason: `no ${policy.workflow} run for ${headSha.slice(0, 7)}` };
|
|
194
|
+
|
|
195
|
+
if (run.status !== "completed") return { status: "pending", run };
|
|
196
|
+
|
|
197
|
+
let text: string | null;
|
|
198
|
+
|
|
199
|
+
try {
|
|
200
|
+
text = deps.report(run.id, policy.artifact, policy.file, policy.max_bytes);
|
|
201
|
+
} catch (e) {
|
|
202
|
+
return { status: "invalid", run, reason: `cannot read ${policy.artifact}: ${e instanceof Error ? e.message.split("\n")[0] : e}` };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (text === null) return { status: "missing", reason: `run ${run.id} uploaded no ${policy.artifact} artifact` };
|
|
206
|
+
|
|
207
|
+
const read = parseReport(text, policy.file);
|
|
208
|
+
|
|
209
|
+
if (!read.ok) return { status: "invalid", run, reason: read.reason };
|
|
210
|
+
|
|
211
|
+
const tested = testedCommit(read.report, headSha, baseSha, deps);
|
|
212
|
+
|
|
213
|
+
if (tested === null) return { status: "invalid", run, reason: `${policy.file} does not record which commit its tests ran on` };
|
|
214
|
+
|
|
215
|
+
return tested === "head" || tested === "merge" ? { status: "ok", run, tested, scope: read.report.execution?.runScope, report: read.report } : { status: "stale", run, tested };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* How many scenarios the head's own run measured checking less than the base branch's: removed, newly skipped, or
|
|
220
|
+
* making fewer steps or assertions. An approval with any of these is held for a person. The report can only hold
|
|
221
|
+
* an approval back, never grant one. Zero unless the evidence is this head's.
|
|
222
|
+
*/
|
|
223
|
+
export const weakenedCount = (e: StoryEvidence | null): number =>
|
|
224
|
+
e?.status === "ok" ? e.report.findings.filter((f) => f.kind === "weakened").length : 0;
|
|
225
|
+
|
|
226
|
+
/** One line for the trusted context: what gate found, never what the report claims. */
|
|
227
|
+
export function formatStoryStatus(e: StoryEvidence): string {
|
|
228
|
+
const head = "Test evidence (executable-stories report from CI; advisory: it never waives a gate or counts as independent assurance, and the PR's author wrote the tests behind it):";
|
|
229
|
+
const run = "run" in e ? ` run ${e.run.id} attempt ${e.run.attempt} (${e.run.event}, ${e.run.conclusion || e.run.status})` : "";
|
|
230
|
+
|
|
231
|
+
switch (e.status) {
|
|
232
|
+
case "missing":
|
|
233
|
+
return `${head} missing, ${e.reason}.`;
|
|
234
|
+
case "pending":
|
|
235
|
+
return `${head}${run} is still running; no report yet.`;
|
|
236
|
+
case "invalid":
|
|
237
|
+
return `${head}${run} report unreadable: ${e.reason}.`;
|
|
238
|
+
case "stale":
|
|
239
|
+
return `${head}${run} tested ${e.tested.slice(0, 12)}, not this head. Read none of it as evidence for this head.`;
|
|
240
|
+
case "ok":
|
|
241
|
+
return `${head}${run} tested ${e.tested === "head" ? "this head" : "the merge of this head into the base"}${e.scope === "filtered" ? " in a filtered run, so it covers only the scenarios that ran" : ""}. The report follows inside the untrusted content.`;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** The report's findings and coverage bands, for inside the untrusted fence. `clip` makes each field one bounded line. */
|
|
246
|
+
export function formatStoryReport(r: ReviewReport, clip: (s: string, n: number) => string): string[] {
|
|
247
|
+
const bands = (band: string) => r.changedFiles.filter((f) => f.band === band).map((f) => clip(f.path, 200));
|
|
248
|
+
|
|
249
|
+
return [
|
|
250
|
+
`Test evidence report: ${r.run.total} scenarios, ${r.run.passed} passed, ${r.run.failed} failed, ${r.run.skipped} skipped, ${r.run.pending} pending.`,
|
|
251
|
+
...r.findings.slice(0, 40).map((f) => ` [${f.severity}] ${f.kind}: ${clip(f.title, 200)}${f.file ? ` (${clip(f.file, 200)}${f.line ? `:${f.line}` : ""})` : ""}: ${clip(f.detail, 500)}`),
|
|
252
|
+
...(bands("uncovered").length ? [` Changed files no scenario covers: ${bands("uncovered").slice(0, 40).join(", ")}`] : []),
|
|
253
|
+
...(bands("weak").length ? [` Changed files only weak scenarios cover: ${bands("weak").slice(0, 40).join(", ")}`] : []),
|
|
254
|
+
];
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** The evidence bundle's record: the status and the run, with counts in place of the report's text. */
|
|
258
|
+
export function storyEvidenceRecord(e: StoryEvidence | null) {
|
|
259
|
+
if (!e) return null;
|
|
260
|
+
|
|
261
|
+
if (e.status === "missing") return { status: e.status, reason: e.reason };
|
|
262
|
+
|
|
263
|
+
const run = { id: e.run.id, url: e.run.url, attempt: e.run.attempt, event: e.run.event, conclusion: e.run.conclusion };
|
|
264
|
+
|
|
265
|
+
switch (e.status) {
|
|
266
|
+
case "pending":
|
|
267
|
+
return { status: e.status, run };
|
|
268
|
+
case "invalid":
|
|
269
|
+
return { status: e.status, run, reason: e.reason };
|
|
270
|
+
case "stale":
|
|
271
|
+
return { status: e.status, run, tested: e.tested };
|
|
272
|
+
case "ok":
|
|
273
|
+
return { status: e.status, run, tested: e.tested, scope: e.scope ?? null, run_counts: e.report.run, findings: e.report.findings.length, uncovered: e.report.changedFiles.filter((f) => f.band === "uncovered").length };
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* One line for the review's test evidence section: counts and status only, never the report's own text. HTML, not
|
|
279
|
+
* Markdown: it sits in a `<summary>`, where GitHub renders no Markdown.
|
|
280
|
+
*/
|
|
281
|
+
export function evidenceSummary(e: StoryEvidence): string {
|
|
282
|
+
const link = (r: StoryRun) => `<a href="${r.url.replace(/[&"<>]/g, (c) => `&#${c.charCodeAt(0)};`)}">run</a>`;
|
|
283
|
+
|
|
284
|
+
if (e.status === "missing") return `none (${e.reason})`;
|
|
285
|
+
|
|
286
|
+
if (e.status === "pending") return `⏳ still running (${link(e.run)})`;
|
|
287
|
+
|
|
288
|
+
if (e.status === "invalid") return `unreadable (${e.reason})`;
|
|
289
|
+
|
|
290
|
+
if (e.status === "stale") return `stale: tested <code>${e.tested.slice(0, 7)}</code> (${link(e.run)})`;
|
|
291
|
+
|
|
292
|
+
const { run } = e.report;
|
|
293
|
+
const weakened = weakenedCount(e);
|
|
294
|
+
const counts = [`${run.passed} passed`, ...(run.failed ? [`❌ ${run.failed} failed`] : []), ...(run.skipped ? [`${run.skipped} skipped`] : [])];
|
|
295
|
+
|
|
296
|
+
return [...counts, ...(weakened ? [`⚠️ ${weakened} scenario${weakened === 1 ? " checks" : "s check"} less than base`] : []), link(e.run)].join(" · ");
|
|
297
|
+
}
|