@niwa-labs/pstack 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +260 -2
- package/package.json +28 -4
- package/scripts/upstream-map.ts +90 -0
- package/src/author.ts +57 -0
- package/src/binding.ts +32 -0
- package/src/catalog.ts +86 -0
- package/src/composition.ts +66 -0
- package/src/index.ts +50 -0
- package/src/mapping/analyze.ts +223 -0
- package/src/mapping/assemble.ts +96 -0
- package/src/mapping/diff.ts +139 -0
- package/src/mapping/format.ts +67 -0
- package/src/mapping/git.ts +69 -0
- package/src/mapping/index.ts +18 -0
- package/src/mapping/maintain.ts +36 -0
- package/src/mapping/schema.ts +50 -0
- package/src/mapping/sources.ts +108 -0
- package/src/mapping/tool.ts +77 -0
- package/src/mapping/verify.ts +130 -0
- package/src/plugin.ts +136 -0
- package/src/policy.ts +115 -0
- package/src/ports.ts +184 -0
- package/src/pr-binding.ts +88 -0
- package/src/records.ts +61 -0
- package/src/roles.ts +49 -0
- package/src/trunk.ts +59 -0
- package/src/upstream-actions.ts +135 -0
- package/src/upstream-records.ts +71 -0
- package/src/upstream.ts +151 -0
- package/src/workflows/adapted.md +780 -0
- package/src/workflows/authoring-a-skill.ts +52 -0
- package/src/workflows/autonomous-run.ts +56 -0
- package/src/workflows/autopilot-full.ts +90 -0
- package/src/workflows/autopilot-stack.ts +125 -0
- package/src/workflows/babysit.ts +124 -0
- package/src/workflows/benny-reproduce.ts +122 -0
- package/src/workflows/benny-triage.ts +89 -0
- package/src/workflows/bug-fix.ts +54 -0
- package/src/workflows/entry.ts +130 -0
- package/src/workflows/eval.ts +63 -0
- package/src/workflows/feature.ts +54 -0
- package/src/workflows/hillclimb.ts +78 -0
- package/src/workflows/investigation.ts +41 -0
- package/src/workflows/lib/catalog.ts +45 -0
- package/src/workflows/lib/change.ts +70 -0
- package/src/workflows/lib/forensics.ts +37 -0
- package/src/workflows/lib/git.ts +30 -0
- package/src/workflows/lib/go.ts +58 -0
- package/src/workflows/lib/inputs.ts +34 -0
- package/src/workflows/lib/lane.ts +170 -0
- package/src/workflows/lib/measure.ts +34 -0
- package/src/workflows/lib/plan.ts +24 -0
- package/src/workflows/lib/script.ts +79 -0
- package/src/workflows/lib/ship.ts +110 -0
- package/src/workflows/lib/texts.ts +18 -0
- package/src/workflows/lib/trunk.ts +91 -0
- package/src/workflows/multi-phase-plan.ts +83 -0
- package/src/workflows/opening-a-pr.ts +62 -0
- package/src/workflows/pause-safely.ts +50 -0
- package/src/workflows/perf-issue.ts +87 -0
- package/src/workflows/prototype.ts +76 -0
- package/src/workflows/refactoring.ts +54 -0
- package/src/workflows/runtime-forensics.ts +26 -0
- package/src/workflows/scenarios.ts +380 -0
- package/src/workflows/session-pickup.ts +57 -0
- package/src/workflows/shipping.ts +63 -0
- package/src/workflows/suite.ts +129 -0
- package/src/workflows/texts/authoring-a-skill.json +5 -0
- package/src/workflows/texts/autonomous-run.json +5 -0
- package/src/workflows/texts/autopilot-full.json +5 -0
- package/src/workflows/texts/autopilot-stack.json +5 -0
- package/src/workflows/texts/babysit.json +5 -0
- package/src/workflows/texts/benny-reproduce.json +5 -0
- package/src/workflows/texts/benny-triage.json +5 -0
- package/src/workflows/texts/bug-fix.json +5 -0
- package/src/workflows/texts/bugbot-triage.json +5 -0
- package/src/workflows/texts/eval.json +5 -0
- package/src/workflows/texts/feature.json +5 -0
- package/src/workflows/texts/hillclimb.json +5 -0
- package/src/workflows/texts/investigation.json +5 -0
- package/src/workflows/texts/multi-phase-plan.json +5 -0
- package/src/workflows/texts/opening-a-pr.json +5 -0
- package/src/workflows/texts/pause-safely.json +5 -0
- package/src/workflows/texts/perf-issue.json +5 -0
- package/src/workflows/texts/prototype.json +5 -0
- package/src/workflows/texts/refactoring.json +5 -0
- package/src/workflows/texts/runtime-forensics.json +5 -0
- package/src/workflows/texts/session-pickup.json +5 -0
- package/src/workflows/texts/trace-forensics.json +5 -0
- package/src/workflows/texts/visual-parity.json +5 -0
- package/src/workflows/texts/worker.json +5 -0
- package/src/workflows/trace-forensics.ts +27 -0
- package/src/workflows/upstream-sync.ts +182 -0
- package/src/workflows/visual-parity.ts +52 -0
- package/src/workflows/worktree-cleanup.ts +57 -0
- package/test/author.test.ts +61 -0
- package/test/benny.test.ts +138 -0
- package/test/binding.test.ts +54 -0
- package/test/catalog-helpers.test.ts +49 -0
- package/test/closure.test.ts +45 -0
- package/test/composition.test.ts +23 -0
- package/test/eval.test.ts +23 -0
- package/test/lanes.test.ts +334 -0
- package/test/mapping-diff.test.ts +64 -0
- package/test/mapping-support.ts +110 -0
- package/test/mapping-texts.test.ts +106 -0
- package/test/mapping.test.ts +276 -0
- package/test/package-layout.ts +68 -0
- package/test/policy.test.ts +98 -0
- package/test/ports.test.ts +45 -0
- package/test/ports.ts +173 -0
- package/test/roles.test.ts +21 -0
- package/test/suite.test.ts +167 -0
- package/test/support.ts +541 -0
- package/test/texts.test.ts +174 -0
- package/test/tool.test.ts +48 -0
- package/test/trunk-helpers.test.ts +149 -0
- package/test/trunk.test.ts +24 -0
- package/test/upstream-sync.test.ts +246 -0
- package/test/upstream.test.ts +106 -0
- package/test/worktree-cleanup.test.ts +107 -0
- package/tsconfig.json +7 -0
- package/upstream/LICENSE +21 -0
- package/upstream/SOURCE.json +5 -0
- package/upstream/benny/FOR_AGENTS.md +89 -0
- package/upstream/benny/README.md +23 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/SKILL.md +310 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/references/control-adapter.md +169 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/references/feature-map.example.md +205 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/references/verify-existing-fix.md +93 -0
- package/upstream/benny/skills/setup-benny/SKILL.md +265 -0
- package/upstream/benny/skills/triage-issue-reports/SKILL.md +240 -0
- package/upstream/benny/skills/triage-issue-reports/references/routing.example.md +61 -0
- package/upstream/benny/templates/configuration.example.yaml +84 -0
- package/upstream/benny/templates/reproduce-automation-prompt.md +33 -0
- package/upstream/benny/templates/triage-automation-prompt.md +39 -0
- package/upstream/mapping.json +699 -0
- package/upstream/poteto-mode/SKILL.md +110 -0
- package/upstream/poteto-mode/playbooks/authoring-a-skill.md +12 -0
- package/upstream/poteto-mode/playbooks/autonomous-run.md +13 -0
- package/upstream/poteto-mode/playbooks/autopilot-full.md +25 -0
- package/upstream/poteto-mode/playbooks/autopilot-stack.md +8 -0
- package/upstream/poteto-mode/playbooks/babysit.md +10 -0
- package/upstream/poteto-mode/playbooks/bug-fix.md +15 -0
- package/upstream/poteto-mode/playbooks/eval.md +25 -0
- package/upstream/poteto-mode/playbooks/feature.md +21 -0
- package/upstream/poteto-mode/playbooks/hillclimb.md +21 -0
- package/upstream/poteto-mode/playbooks/investigation.md +14 -0
- package/upstream/poteto-mode/playbooks/multi-phase-plan.md +11 -0
- package/upstream/poteto-mode/playbooks/opening-a-pr.md +9 -0
- package/upstream/poteto-mode/playbooks/pause-safely.md +10 -0
- package/upstream/poteto-mode/playbooks/perf-issue.md +25 -0
- package/upstream/poteto-mode/playbooks/prototype.md +14 -0
- package/upstream/poteto-mode/playbooks/refactoring.md +16 -0
- package/upstream/poteto-mode/playbooks/runtime-forensics.md +11 -0
- package/upstream/poteto-mode/playbooks/session-pickup.md +11 -0
- package/upstream/poteto-mode/playbooks/shipping.md +12 -0
- package/upstream/poteto-mode/playbooks/trace-forensics.md +14 -0
- package/upstream/poteto-mode/playbooks/visual-parity.md +11 -0
- package/upstream/poteto-mode/playbooks/worktree-cleanup.md +12 -0
- package/upstream/poteto-mode/references/bugbot-triage.md +142 -0
- package/upstream/poteto-mode/worker.md +11 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Clean up the registered checkout of the pull request named by an observed merge.
|
|
2
|
+
import { PrMerged } from "@niwa-labs/forge/records";
|
|
3
|
+
import type { ScriptApi } from "@niwa-labs/script";
|
|
4
|
+
|
|
5
|
+
import { isRefusal } from "./lib/catalog.ts";
|
|
6
|
+
import { api } from "./lib/script.ts";
|
|
7
|
+
|
|
8
|
+
declare const workflow: ScriptApi["workflow"];
|
|
9
|
+
declare const z: ScriptApi["z"];
|
|
10
|
+
|
|
11
|
+
interface MergeTrigger {
|
|
12
|
+
id: string;
|
|
13
|
+
type: string;
|
|
14
|
+
data: unknown;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
interface Binding {
|
|
18
|
+
worktree: { id: string; repo: string; path: string; branch: string };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export default workflow(
|
|
22
|
+
{
|
|
23
|
+
name: "pstack-worktree-cleanup",
|
|
24
|
+
description: "Remove the registered checkout of a merged pull request",
|
|
25
|
+
on: ["forge.pr.merged"],
|
|
26
|
+
output: z.strictObject({ removed: z.array(z.string()), kept: z.array(z.strictObject({ id: z.string(), why: z.string() })) }),
|
|
27
|
+
uses: ["niwa.records.read", "pstack.pr.binding", "git.worktree.remove"],
|
|
28
|
+
},
|
|
29
|
+
async ({ id, type, data }: MergeTrigger) => {
|
|
30
|
+
const skipped = { removed: [], kept: [] };
|
|
31
|
+
const merged = PrMerged.schema.safeParse(data);
|
|
32
|
+
if (type !== PrMerged.name || !merged.success || merged.data.head === undefined) return skipped;
|
|
33
|
+
const { head, pr: { repo: { host, path }, number } } = merged.data;
|
|
34
|
+
const repo = { host, path };
|
|
35
|
+
|
|
36
|
+
// A caller may start this workflow with its own arguments. Require the observation that the forge wrote.
|
|
37
|
+
const { records } = await api().read.records(PrMerged, {
|
|
38
|
+
filter: { "pr.repo.host": host, "pr.repo.path": path, "pr.number": number, head },
|
|
39
|
+
}, { label: "merge" });
|
|
40
|
+
const observed = records.find((row) => row.id === id && row.actor.kind === "host");
|
|
41
|
+
if (observed === undefined) return skipped;
|
|
42
|
+
|
|
43
|
+
const { binding } = (await api().call("pstack.pr.binding", { repo, number, head, before: observed.seq }, { label: "binding" })) as { binding?: Binding };
|
|
44
|
+
if (binding === undefined) return skipped;
|
|
45
|
+
const { id: worktree, repo: local, path: checkout, branch } = binding.worktree;
|
|
46
|
+
try {
|
|
47
|
+
const result = (await api().call("git.worktree.remove", {
|
|
48
|
+
id: worktree,
|
|
49
|
+
onlyIf: { repo: local, path: checkout, branch, head, project: repo },
|
|
50
|
+
}, { label: "remove" })) as { id: string; removed: boolean };
|
|
51
|
+
return result.removed && result.id === worktree ? { removed: [worktree], kept: [] } : skipped;
|
|
52
|
+
} catch (error) {
|
|
53
|
+
if (!isRefusal(error)) throw error;
|
|
54
|
+
return { removed: [], kept: [{ id: worktree, why: (error as Error).message }] };
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
);
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// The author's box: a session of the author writes only in one of niwa's worktrees and reaches only pstack's forges.
|
|
2
|
+
|
|
3
|
+
import { expect, test } from "bun:test";
|
|
4
|
+
import { mkdirSync, realpathSync, symlinkSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
import { tempDir } from "@niwa-labs/testing";
|
|
8
|
+
|
|
9
|
+
import { AuthorRefusal, authorLaunch } from "../src/author.ts";
|
|
10
|
+
|
|
11
|
+
const author = { harness: "author", box: true as const };
|
|
12
|
+
|
|
13
|
+
async function place(): Promise<{ worktrees: string; wt: string; outside: string }> {
|
|
14
|
+
const root = realpathSync(await tempDir("author"));
|
|
15
|
+
const worktrees = join(root, "worktrees");
|
|
16
|
+
const wt = join(worktrees, "wt-1");
|
|
17
|
+
mkdirSync(wt, { recursive: true });
|
|
18
|
+
const outside = join(root, "repo");
|
|
19
|
+
mkdirSync(outside);
|
|
20
|
+
return { worktrees, wt, outside };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const code = (fn: () => unknown): string | undefined => {
|
|
24
|
+
try {
|
|
25
|
+
fn();
|
|
26
|
+
return undefined;
|
|
27
|
+
} catch (error) {
|
|
28
|
+
return error instanceof AuthorRefusal ? error.code : String(error);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
test("the author writes in its worktree and reaches only the forges", async () => {
|
|
33
|
+
const { worktrees, wt } = await place();
|
|
34
|
+
expect(authorLaunch(author, wt, worktrees, ["gitlab.example.com"])).toEqual({
|
|
35
|
+
place: wt,
|
|
36
|
+
box: { allow: ["project", { net: "gitlab.example.com" }] },
|
|
37
|
+
});
|
|
38
|
+
expect(authorLaunch(author, wt, worktrees, [])).toEqual({ place: wt, box: { allow: ["project"] } });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("a place that is not one of niwa's worktrees is refused", async () => {
|
|
42
|
+
const { worktrees, wt, outside } = await place();
|
|
43
|
+
expect(code(() => authorLaunch(author, outside, worktrees, []))).toBe("not-a-worktree");
|
|
44
|
+
expect(code(() => authorLaunch(author, worktrees, worktrees, []))).toBe("not-a-worktree");
|
|
45
|
+
expect(code(() => authorLaunch(author, join(wt, "src"), worktrees, []))).toBe("no-place");
|
|
46
|
+
mkdirSync(join(wt, "src"));
|
|
47
|
+
expect(code(() => authorLaunch(author, join(wt, "src"), worktrees, []))).toBe("not-a-worktree");
|
|
48
|
+
expect(code(() => authorLaunch(author, "", worktrees, []))).toBe("no-place");
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("a link into a worktree from outside is followed to where it leads", async () => {
|
|
52
|
+
const { worktrees, outside } = await place();
|
|
53
|
+
const link = join(worktrees, "wt-2");
|
|
54
|
+
symlinkSync(outside, link, "dir");
|
|
55
|
+
expect(code(() => authorLaunch(author, link, worktrees, []))).toBe("not-a-worktree");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("an author role with no box is refused", async () => {
|
|
59
|
+
const { worktrees, wt } = await place();
|
|
60
|
+
expect(code(() => authorLaunch({ harness: "author" } as never, wt, worktrees, []))).toBe("no-box");
|
|
61
|
+
});
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// Benny, when enabled: an agent for the agents module, and two gated automations. A report handed to him is triaged;
|
|
2
|
+
// a bug is reproduced and fixed in a draft pull request; nothing else is. Before the suite promoted them, a report
|
|
3
|
+
// starts nothing.
|
|
4
|
+
|
|
5
|
+
import { afterEach, expect, test } from "bun:test";
|
|
6
|
+
|
|
7
|
+
import { eventually } from "@niwa-labs/testing";
|
|
8
|
+
|
|
9
|
+
import { BENNY, UNGATED, gatedOf } from "../src/catalog.ts";
|
|
10
|
+
import { builtins, defaultAnswers, pstackHost, recordsOf, registeredAll } from "./support.ts";
|
|
11
|
+
import type { PstackHost } from "./support.ts";
|
|
12
|
+
|
|
13
|
+
const hosts: PstackHost[] = [];
|
|
14
|
+
afterEach(async () => {
|
|
15
|
+
for (const one of hosts.splice(0)) await one.close();
|
|
16
|
+
}, 30_000);
|
|
17
|
+
|
|
18
|
+
async function served(benny = true): Promise<PstackHost> {
|
|
19
|
+
const made = await pstackHost(benny ? {} : { benny: false });
|
|
20
|
+
hosts.push(made);
|
|
21
|
+
made.harnesses.answers = defaultAnswers(made);
|
|
22
|
+
await registeredAll(made.host, gatedOf(benny).length + UNGATED.length);
|
|
23
|
+
return made;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async function promote(made: PstackHost): Promise<void> {
|
|
27
|
+
const suite = (await builtins(made.host))["suite"];
|
|
28
|
+
const ran = (await made.host.call("workflow.start" as never, { id: suite?.id, args: made.fixture, wait: true, timeoutMs: 120_000 } as never)) as { outcome?: { result?: { status: string } } };
|
|
29
|
+
expect(ran.outcome?.result?.status).toBe("promoted");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const runsOf = async (made: PstackHost, name: string) => {
|
|
33
|
+
const id = (await builtins(made.host))[name]?.id;
|
|
34
|
+
const requested = new Set((await recordsOf(made.host, "workflow.run.requested")).filter((one) => (one.data as { workflow?: string }).workflow === id).map((one) => (one.data as { run: string }).run));
|
|
35
|
+
return (await recordsOf(made.host, "workflow.run.finished")).filter((one) => requested.has((one.data as { run: string }).run)).map((one) => (one.data as { outcome: { result?: any } }).outcome.result);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
test("a report starts nothing before the suite promoted Benny, and is triaged and fixed in a draft after", async () => {
|
|
39
|
+
const made = await served();
|
|
40
|
+
expect(made.agents.map((one) => one.name)).toEqual(["benny"]);
|
|
41
|
+
await made.asOwner("benny.report", { report: "The README calls the service payroll." });
|
|
42
|
+
await Bun.sleep(1000);
|
|
43
|
+
expect(await recordsOf(made.host, "workflow.run.requested")).toEqual([]);
|
|
44
|
+
|
|
45
|
+
await promote(made);
|
|
46
|
+
// The suite's own scenarios ran each automation once.
|
|
47
|
+
const before = { mrs: made.project.mrs.size, triaged: (await runsOf(made, "benny-triage")).length, fixed: (await runsOf(made, "benny-reproduce")).length };
|
|
48
|
+
await made.asOwner("benny.report", { report: "The README calls the service payroll.", from: "a customer" });
|
|
49
|
+
await eventually(async () => (await runsOf(made, "benny-reproduce")).length === before.fixed + 1, 60_000, "the bug was not reproduced");
|
|
50
|
+
const triaged = (await runsOf(made, "benny-triage")).slice(before.triaged);
|
|
51
|
+
expect(triaged).toEqual([expect.objectContaining({ verdict: "bug" })]);
|
|
52
|
+
const fixed = (await runsOf(made, "benny-reproduce")).at(-1);
|
|
53
|
+
expect(fixed).toMatchObject({ status: "verified", runs: ["The README says payroll.", "It still says payroll."] });
|
|
54
|
+
expect(made.project.mrs.size).toBe(before.mrs + 1);
|
|
55
|
+
const report = made.harnesses.of("benny-triage:triage").at(-1)?.prompt ?? "";
|
|
56
|
+
expect(report).toContain("<report>\nThe README calls the service payroll.\n</report>");
|
|
57
|
+
// This report came with no source: the session is told so, and that no issue is filed for it.
|
|
58
|
+
expect(report).toContain("This report has no source: file no issue for it");
|
|
59
|
+
}, 240_000);
|
|
60
|
+
|
|
61
|
+
test("a report that is not a bug is triaged and goes no further", async () => {
|
|
62
|
+
const made = await served();
|
|
63
|
+
await promote(made);
|
|
64
|
+
made.harnesses.answers["benny-triage:triage"] = () => ({ verdict: "other", owner: "nobody", summary: "A question." });
|
|
65
|
+
const reproduced = (await runsOf(made, "benny-reproduce")).length;
|
|
66
|
+
await made.asOwner("benny.report", { report: "How do I read the README?" });
|
|
67
|
+
await eventually(async () => (await runsOf(made, "benny-triage")).some((one) => one?.verdict === "other"), 60_000, "the report was not triaged");
|
|
68
|
+
await Bun.sleep(1000);
|
|
69
|
+
expect((await runsOf(made, "benny-reproduce")).length).toBe(reproduced);
|
|
70
|
+
}, 240_000);
|
|
71
|
+
|
|
72
|
+
test("without Benny there is no agent, no report action and none of his automations, and the suite still promotes", async () => {
|
|
73
|
+
const made = await served(false);
|
|
74
|
+
expect(made.agents).toEqual([]);
|
|
75
|
+
const all = await builtins(made.host);
|
|
76
|
+
for (const { name } of BENNY) expect(all[name]).toBeUndefined();
|
|
77
|
+
await expect(made.asOwner("benny.report", { report: "x" })).rejects.toThrow();
|
|
78
|
+
await promote(made);
|
|
79
|
+
}, 240_000);
|
|
80
|
+
|
|
81
|
+
test("a report with a source gets its verdict once, in its own thread, through the channel, and the session is shown the tracker, not the channel", async () => {
|
|
82
|
+
const made = await served();
|
|
83
|
+
await promote(made);
|
|
84
|
+
made.ports.calls.length = 0;
|
|
85
|
+
const before = (await runsOf(made, "benny-reproduce")).length;
|
|
86
|
+
made.harnesses.answers["benny-triage:triage"] = async () => {
|
|
87
|
+
const created = (await made.asOwner("benny.tracker.create", { title: "README names the wrong service", body: "Quoted.", source: "thread-9" })) as { id: string; url: string };
|
|
88
|
+
return { verdict: "bug", owner: "the README", summary: "The README names the wrong service.", tracker: created };
|
|
89
|
+
};
|
|
90
|
+
await made.asOwner("benny.report", { report: "The README calls the service payroll.", from: "a customer", source: "thread-9" });
|
|
91
|
+
await eventually(async () => (await runsOf(made, "benny-reproduce")).length === before + 1, 60_000, "the bug was not reproduced");
|
|
92
|
+
|
|
93
|
+
const replies = made.ports.of("benny.channel.reply");
|
|
94
|
+
expect(replies).toHaveLength(1);
|
|
95
|
+
expect(replies[0]?.input).toEqual({ source: "thread-9", text: "bug: The README names the wrong service.\nhttps://tracker.example/issue/ISSUE-1" });
|
|
96
|
+
const triage = made.harnesses.of("benny-triage:triage").at(-1);
|
|
97
|
+
expect(triage?.visible).toEqual(["benny.tracker.find", "benny.tracker.create", "benny.tracker.comment"]);
|
|
98
|
+
expect(triage?.prompt).toContain("The report's source: thread-9");
|
|
99
|
+
expect(made.ports.issues.get("ISSUE-1")?.prs).toEqual([expect.stringMatching(/^gitlab\.example\.com\/acme\/billing\/pr\/\d+$/)]);
|
|
100
|
+
expect(made.ports.of("benny.tracker.link-pr")).toHaveLength(1);
|
|
101
|
+
expect(made.ports.of("benny.channel.notify")).toEqual([expect.objectContaining({ input: expect.objectContaining({ source: "thread-9" }) })]);
|
|
102
|
+
}, 240_000);
|
|
103
|
+
|
|
104
|
+
test("the reproduction session is shown the control actions of the product, and a report with no source posts nowhere", async () => {
|
|
105
|
+
const made = await served();
|
|
106
|
+
await promote(made);
|
|
107
|
+
made.ports.calls.length = 0;
|
|
108
|
+
const before = (await runsOf(made, "benny-reproduce")).length;
|
|
109
|
+
await made.asOwner("benny.report", { report: "The README calls the service payroll." });
|
|
110
|
+
await eventually(async () => (await runsOf(made, "benny-reproduce")).length === before + 1, 60_000, "the bug was not reproduced");
|
|
111
|
+
|
|
112
|
+
const repro = made.harnesses.of("benny-reproduce:reproduce").at(-1);
|
|
113
|
+
expect(repro?.visible).toEqual([
|
|
114
|
+
"benny.control.capabilities",
|
|
115
|
+
"benny.control.bring-up",
|
|
116
|
+
"benny.control.drive",
|
|
117
|
+
"benny.control.inspect",
|
|
118
|
+
"benny.control.screenshot",
|
|
119
|
+
"benny.control.record",
|
|
120
|
+
"benny.control.cleanup",
|
|
121
|
+
]);
|
|
122
|
+
expect(repro?.prompt).toContain("benny.control.capabilities");
|
|
123
|
+
expect(made.ports.of("benny.channel.reply")).toEqual([]);
|
|
124
|
+
expect(made.ports.of("benny.channel.notify")).toEqual([]);
|
|
125
|
+
expect(made.ports.of("benny.tracker.link-pr")).toEqual([]);
|
|
126
|
+
}, 240_000);
|
|
127
|
+
|
|
128
|
+
test("a report whose bug is not reproduced is told to the channel once, and no pull request is linked", async () => {
|
|
129
|
+
const made = await served();
|
|
130
|
+
await promote(made);
|
|
131
|
+
made.ports.calls.length = 0;
|
|
132
|
+
made.harnesses.answers["benny-reproduce:reproduce"] = () => ({ reproduced: false, runs: ["The README says billing."] });
|
|
133
|
+
const before = (await runsOf(made, "benny-reproduce")).length;
|
|
134
|
+
await made.asOwner("benny.report", { report: "The README calls the service payroll.", source: "thread-3" });
|
|
135
|
+
await eventually(async () => (await runsOf(made, "benny-reproduce")).length === before + 1, 60_000, "the bug was not reproduced");
|
|
136
|
+
expect(made.ports.of("benny.channel.notify")).toEqual([expect.objectContaining({ input: { text: "Could not reproduce: The README names the wrong service.", source: "thread-3" } })]);
|
|
137
|
+
expect(made.ports.of("benny.tracker.link-pr")).toEqual([]);
|
|
138
|
+
}, 240_000);
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// The binding evidence attests with: a session keeps one address across a relaunch under a new token, an agent's
|
|
2
|
+
// session attests as the agent, and the owner and runs attest as the evidence module binds them.
|
|
3
|
+
|
|
4
|
+
import { expect, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
import { principalOf } from "@niwa-labs/core";
|
|
7
|
+
import type { Caller } from "@niwa-labs/core";
|
|
8
|
+
import { plugin } from "@niwa-labs/plugin";
|
|
9
|
+
import type { CallerBinding } from "@niwa-labs/plugin";
|
|
10
|
+
|
|
11
|
+
import { addressingOf } from "../src/binding.ts";
|
|
12
|
+
|
|
13
|
+
/** A sessions module whose tokens' marks name sessions: two marks of one session are one relaunched session. */
|
|
14
|
+
function sessions(marks: Record<string, string>) {
|
|
15
|
+
return Object.assign(plugin("sessions", () => {}), {
|
|
16
|
+
bind: (fallback?: CallerBinding): CallerBinding => (caller) => {
|
|
17
|
+
const principal = caller.kind === "client" ? caller.principal : undefined;
|
|
18
|
+
const session = principal?.kind === "session" ? marks[principal.id] : undefined;
|
|
19
|
+
if (session === undefined) return fallback?.(caller);
|
|
20
|
+
const [harness, id] = session.split(" ") as [string, string];
|
|
21
|
+
return { kind: "session", harness, id };
|
|
22
|
+
},
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** An agents module whose agent `reviewer` works through the session `verifier n-9`. */
|
|
27
|
+
function agents() {
|
|
28
|
+
return Object.assign(plugin("agents", () => {}), {
|
|
29
|
+
bind: (fallback: CallerBinding): CallerBinding => (caller) => {
|
|
30
|
+
const bound = fallback(caller);
|
|
31
|
+
return bound?.kind === "session" && bound.harness === "verifier" && bound.id === "n-9" ? { kind: "agent", id: "reviewer" } : bound;
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const client = (kind: "session" | "run" | "owner", id: string): Caller => ({ kind: "client", id: `c-${id}` as never, principal: principalOf(kind, id) });
|
|
37
|
+
|
|
38
|
+
test("a session relaunched under a new token keeps its address", () => {
|
|
39
|
+
const bind = addressingOf([sessions({ "mark-1": "author n-1", "mark-2": "author n-1" })]);
|
|
40
|
+
expect(bind(client("session", "mark-1"))).toEqual({ kind: "session", harness: "author", id: "n-1" });
|
|
41
|
+
expect(bind(client("session", "mark-2"))).toEqual(bind(client("session", "mark-1")));
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("an agent's session attests as the agent; the owner and a run as evidence binds them", () => {
|
|
45
|
+
const bind = addressingOf([sessions({ "mark-3": "verifier n-9" }), agents()]);
|
|
46
|
+
expect(bind(client("session", "mark-3"))).toEqual({ kind: "agent", id: "reviewer" });
|
|
47
|
+
expect(bind(client("owner", "owner"))).toEqual({ kind: "owner" });
|
|
48
|
+
expect(bind(client("run", "run-1"))).toEqual({ kind: "client", id: "run:run-1" });
|
|
49
|
+
expect(bind({ kind: "host" })).toBeUndefined();
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("a host with no sessions module is refused", () => {
|
|
53
|
+
expect(() => addressingOf([agents()])).toThrow("no sessions module");
|
|
54
|
+
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// How the suite starts a candidate: a start the host refuses is the scenario's failure, but a signal of the runtime
|
|
2
|
+
// (a step that failed pauses the run, a cancel ends it) is not caught: the suite's run pauses or ends as it should.
|
|
3
|
+
|
|
4
|
+
import { afterEach, expect, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
import { isRefusal, tryVersion } from "../src/workflows/lib/catalog.ts";
|
|
7
|
+
|
|
8
|
+
const globals = globalThis as Record<string, unknown>;
|
|
9
|
+
afterEach(() => {
|
|
10
|
+
for (const name of ["call", "ask", "session"]) delete globals[name];
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
/** An error of the name the script runtime gives it. */
|
|
14
|
+
const named = (name: string, message: string) => Object.assign(new Error(message), { name });
|
|
15
|
+
|
|
16
|
+
function startsWith(fail: Error | undefined, answer: unknown = {}): string[] {
|
|
17
|
+
const calls: string[] = [];
|
|
18
|
+
globals.ask = async () => ({});
|
|
19
|
+
globals.session = async () => ({});
|
|
20
|
+
globals.call = async (name: string) => {
|
|
21
|
+
calls.push(name);
|
|
22
|
+
if (fail !== undefined) throw fail;
|
|
23
|
+
return answer;
|
|
24
|
+
};
|
|
25
|
+
return calls;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
test("a refused start is the scenario's failure", async () => {
|
|
29
|
+
startsWith(named("RunRefusal", "the version is a candidate"));
|
|
30
|
+
expect(await tryVersion("t", "wf-1", "v1", {})).toEqual({ ok: false, error: { name: "RunRefusal", message: "the version is a candidate" } });
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("a step that failed or a cancel is not caught", async () => {
|
|
34
|
+
for (const signal of ["PauseSignal", "Cancelled"]) {
|
|
35
|
+
startsWith(named(signal, "step failed"));
|
|
36
|
+
await expect(tryVersion("t", "wf-1", "v1", {})).rejects.toThrow("step failed");
|
|
37
|
+
expect(isRefusal(named(signal, "x"))).toBe(false);
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("a run that did not end in time is cancelled, and one that ended with no result failed", async () => {
|
|
42
|
+
const calls = startsWith(undefined, { run: "run-1", state: "paused", reason: "a step failed" });
|
|
43
|
+
expect(await tryVersion("t", "wf-1", "v1", {})).toEqual({ ok: false, error: { name: "NotEnded", message: "the run was paused (a step failed) and was cancelled" } });
|
|
44
|
+
expect(calls).toEqual(["workflow.start", "workflow.cancel"]);
|
|
45
|
+
startsWith(undefined, { run: "run-2", state: "finished", outcome: { status: "cancelled" } });
|
|
46
|
+
expect(await tryVersion("t", "wf-1", "v1", {})).toEqual({ ok: false, error: { name: "Ended", message: "the run ended cancelled" } });
|
|
47
|
+
startsWith(undefined, { run: "run-3", state: "finished", outcome: { status: "done", result: 7 } });
|
|
48
|
+
expect(await tryVersion("t", "wf-1", "v1", {})).toEqual({ ok: true, value: 7 });
|
|
49
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// A built-in is stored as the files its imports reach, and the host reaches them as the runtime does: an import of
|
|
2
|
+
// types alone is erased first, so a file only types are taken from is not stored, and the stored script does not
|
|
3
|
+
// pass its check. Every local file a script names, for its types too, must also be reached by a value import.
|
|
4
|
+
|
|
5
|
+
import { expect, test } from "bun:test";
|
|
6
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
7
|
+
import { dirname, resolve } from "node:path";
|
|
8
|
+
|
|
9
|
+
import * as ts from "typescript";
|
|
10
|
+
|
|
11
|
+
import { GATED, UNGATED, scriptOf } from "../src/catalog.ts";
|
|
12
|
+
|
|
13
|
+
const transpiler = new Bun.Transpiler({ loader: "tsx" });
|
|
14
|
+
|
|
15
|
+
const local = (from: string, specifier: string): string | undefined => {
|
|
16
|
+
if (!specifier.startsWith(".")) return undefined;
|
|
17
|
+
const base = resolve(dirname(from), specifier);
|
|
18
|
+
return [base, `${base}.ts`].find((one) => existsSync(one));
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/** The files a script reaches: by value imports alone (what is stored), or by every import (what its check reads). */
|
|
22
|
+
function reached(entry: string, all: boolean): Set<string> {
|
|
23
|
+
const seen = new Set<string>();
|
|
24
|
+
const pending = [entry];
|
|
25
|
+
for (let next = pending.pop(); next !== undefined; next = pending.pop()) {
|
|
26
|
+
if (seen.has(next)) continue;
|
|
27
|
+
seen.add(next);
|
|
28
|
+
if (!next.endsWith(".ts")) continue;
|
|
29
|
+
const text = readFileSync(next, "utf8");
|
|
30
|
+
const specifiers = all ? ts.preProcessFile(text, true, true).importedFiles.map((one) => one.fileName) : transpiler.scanImports(text).map((one) => one.path);
|
|
31
|
+
for (const specifier of specifiers) {
|
|
32
|
+
const found = local(next, specifier);
|
|
33
|
+
if (found !== undefined) pending.push(found);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return seen;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
test("every local file a built-in names is stored with it", () => {
|
|
40
|
+
for (const { name } of [...GATED, ...UNGATED]) {
|
|
41
|
+
const stored = reached(scriptOf(name), false);
|
|
42
|
+
const named = [...reached(scriptOf(name), true)].filter((file) => !stored.has(file));
|
|
43
|
+
expect({ name, missing: named }).toEqual({ name, missing: [] });
|
|
44
|
+
}
|
|
45
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// pstack's composition: the plugins it installs, in order, with the author's network held to the forges' hosts.
|
|
2
|
+
|
|
3
|
+
import { expect, test } from "bun:test";
|
|
4
|
+
|
|
5
|
+
import { secret } from "@niwa-labs/forge";
|
|
6
|
+
|
|
7
|
+
import { pstack } from "../src/composition.ts";
|
|
8
|
+
import { ROLES } from "./support.ts";
|
|
9
|
+
|
|
10
|
+
const forge = { servers: [{ host: "gitlab.example.com", url: "https://gitlab.example.com", token: secret("GITLAB_TOKEN") }], poll: false as const };
|
|
11
|
+
|
|
12
|
+
test("the composition installs forge, evidence, watch and plan before pstack", () => {
|
|
13
|
+
const plugins = pstack({ bind: () => undefined, forge, roles: ROLES });
|
|
14
|
+
expect(plugins.map((one) => String(one.name))).toEqual(["forge", "evidence", "watch", "plan", "pstack"]);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("the composition refuses to be made without the host's binding", () => {
|
|
18
|
+
expect(() => pstack({ forge, roles: ROLES } as never)).toThrow("caller binding");
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("an author role without a box refuses the composition", () => {
|
|
22
|
+
expect(() => pstack({ bind: () => undefined, forge, roles: { ...ROLES, author: { harness: "author" } } as never })).toThrow("box");
|
|
23
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { expect, test } from "bun:test";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { ROLES, defaultAnswers, pstackHost } from "./support.ts";
|
|
5
|
+
|
|
6
|
+
test("the eval workflow returns a null score for a variant without an independent grader", async () => {
|
|
7
|
+
const made = await pstackHost({ benny: false, roles: { ...ROLES, judge: { harness: "judge", vendor: "judge" } } });
|
|
8
|
+
try {
|
|
9
|
+
made.harnesses.answers = defaultAnswers(made);
|
|
10
|
+
const ran = await made.host.call("workflow.start" as never, {
|
|
11
|
+
file: join(import.meta.dir, "../src/workflows/eval.ts"),
|
|
12
|
+
args: { ...made.fixture, task: "Compare two replies", variants: { independent: "reviewer", ungraded: "judge" } },
|
|
13
|
+
wait: true,
|
|
14
|
+
timeoutMs: 30_000,
|
|
15
|
+
} as never) as { outcome?: unknown };
|
|
16
|
+
expect(ran.outcome).toMatchObject({
|
|
17
|
+
status: "done",
|
|
18
|
+
result: { status: "judged", table: { independent: { readme: 1 }, ungraded: { readme: null } }, recommend: null },
|
|
19
|
+
});
|
|
20
|
+
} finally {
|
|
21
|
+
await made.close();
|
|
22
|
+
}
|
|
23
|
+
}, 60_000);
|