@niwa-labs/pstack 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +260 -2
- package/package.json +28 -4
- package/scripts/upstream-map.ts +90 -0
- package/src/author.ts +57 -0
- package/src/binding.ts +32 -0
- package/src/catalog.ts +86 -0
- package/src/composition.ts +66 -0
- package/src/index.ts +50 -0
- package/src/mapping/analyze.ts +223 -0
- package/src/mapping/assemble.ts +96 -0
- package/src/mapping/diff.ts +139 -0
- package/src/mapping/format.ts +67 -0
- package/src/mapping/git.ts +69 -0
- package/src/mapping/index.ts +18 -0
- package/src/mapping/maintain.ts +36 -0
- package/src/mapping/schema.ts +50 -0
- package/src/mapping/sources.ts +108 -0
- package/src/mapping/tool.ts +77 -0
- package/src/mapping/verify.ts +130 -0
- package/src/plugin.ts +136 -0
- package/src/policy.ts +115 -0
- package/src/ports.ts +184 -0
- package/src/pr-binding.ts +88 -0
- package/src/records.ts +61 -0
- package/src/roles.ts +49 -0
- package/src/trunk.ts +59 -0
- package/src/upstream-actions.ts +135 -0
- package/src/upstream-records.ts +71 -0
- package/src/upstream.ts +151 -0
- package/src/workflows/adapted.md +780 -0
- package/src/workflows/authoring-a-skill.ts +52 -0
- package/src/workflows/autonomous-run.ts +56 -0
- package/src/workflows/autopilot-full.ts +90 -0
- package/src/workflows/autopilot-stack.ts +125 -0
- package/src/workflows/babysit.ts +124 -0
- package/src/workflows/benny-reproduce.ts +122 -0
- package/src/workflows/benny-triage.ts +89 -0
- package/src/workflows/bug-fix.ts +54 -0
- package/src/workflows/entry.ts +130 -0
- package/src/workflows/eval.ts +63 -0
- package/src/workflows/feature.ts +54 -0
- package/src/workflows/hillclimb.ts +78 -0
- package/src/workflows/investigation.ts +41 -0
- package/src/workflows/lib/catalog.ts +45 -0
- package/src/workflows/lib/change.ts +70 -0
- package/src/workflows/lib/forensics.ts +37 -0
- package/src/workflows/lib/git.ts +30 -0
- package/src/workflows/lib/go.ts +58 -0
- package/src/workflows/lib/inputs.ts +34 -0
- package/src/workflows/lib/lane.ts +170 -0
- package/src/workflows/lib/measure.ts +34 -0
- package/src/workflows/lib/plan.ts +24 -0
- package/src/workflows/lib/script.ts +79 -0
- package/src/workflows/lib/ship.ts +110 -0
- package/src/workflows/lib/texts.ts +18 -0
- package/src/workflows/lib/trunk.ts +91 -0
- package/src/workflows/multi-phase-plan.ts +83 -0
- package/src/workflows/opening-a-pr.ts +62 -0
- package/src/workflows/pause-safely.ts +50 -0
- package/src/workflows/perf-issue.ts +87 -0
- package/src/workflows/prototype.ts +76 -0
- package/src/workflows/refactoring.ts +54 -0
- package/src/workflows/runtime-forensics.ts +26 -0
- package/src/workflows/scenarios.ts +380 -0
- package/src/workflows/session-pickup.ts +57 -0
- package/src/workflows/shipping.ts +63 -0
- package/src/workflows/suite.ts +129 -0
- package/src/workflows/texts/authoring-a-skill.json +5 -0
- package/src/workflows/texts/autonomous-run.json +5 -0
- package/src/workflows/texts/autopilot-full.json +5 -0
- package/src/workflows/texts/autopilot-stack.json +5 -0
- package/src/workflows/texts/babysit.json +5 -0
- package/src/workflows/texts/benny-reproduce.json +5 -0
- package/src/workflows/texts/benny-triage.json +5 -0
- package/src/workflows/texts/bug-fix.json +5 -0
- package/src/workflows/texts/bugbot-triage.json +5 -0
- package/src/workflows/texts/eval.json +5 -0
- package/src/workflows/texts/feature.json +5 -0
- package/src/workflows/texts/hillclimb.json +5 -0
- package/src/workflows/texts/investigation.json +5 -0
- package/src/workflows/texts/multi-phase-plan.json +5 -0
- package/src/workflows/texts/opening-a-pr.json +5 -0
- package/src/workflows/texts/pause-safely.json +5 -0
- package/src/workflows/texts/perf-issue.json +5 -0
- package/src/workflows/texts/prototype.json +5 -0
- package/src/workflows/texts/refactoring.json +5 -0
- package/src/workflows/texts/runtime-forensics.json +5 -0
- package/src/workflows/texts/session-pickup.json +5 -0
- package/src/workflows/texts/trace-forensics.json +5 -0
- package/src/workflows/texts/visual-parity.json +5 -0
- package/src/workflows/texts/worker.json +5 -0
- package/src/workflows/trace-forensics.ts +27 -0
- package/src/workflows/upstream-sync.ts +182 -0
- package/src/workflows/visual-parity.ts +52 -0
- package/src/workflows/worktree-cleanup.ts +57 -0
- package/test/author.test.ts +61 -0
- package/test/benny.test.ts +138 -0
- package/test/binding.test.ts +54 -0
- package/test/catalog-helpers.test.ts +49 -0
- package/test/closure.test.ts +45 -0
- package/test/composition.test.ts +23 -0
- package/test/eval.test.ts +23 -0
- package/test/lanes.test.ts +334 -0
- package/test/mapping-diff.test.ts +64 -0
- package/test/mapping-support.ts +110 -0
- package/test/mapping-texts.test.ts +106 -0
- package/test/mapping.test.ts +276 -0
- package/test/package-layout.ts +68 -0
- package/test/policy.test.ts +98 -0
- package/test/ports.test.ts +45 -0
- package/test/ports.ts +173 -0
- package/test/roles.test.ts +21 -0
- package/test/suite.test.ts +167 -0
- package/test/support.ts +541 -0
- package/test/texts.test.ts +174 -0
- package/test/tool.test.ts +48 -0
- package/test/trunk-helpers.test.ts +149 -0
- package/test/trunk.test.ts +24 -0
- package/test/upstream-sync.test.ts +246 -0
- package/test/upstream.test.ts +106 -0
- package/test/worktree-cleanup.test.ts +107 -0
- package/tsconfig.json +7 -0
- package/upstream/LICENSE +21 -0
- package/upstream/SOURCE.json +5 -0
- package/upstream/benny/FOR_AGENTS.md +89 -0
- package/upstream/benny/README.md +23 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/SKILL.md +310 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/references/control-adapter.md +169 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/references/feature-map.example.md +205 -0
- package/upstream/benny/skills/reproduce-and-fix-issues/references/verify-existing-fix.md +93 -0
- package/upstream/benny/skills/setup-benny/SKILL.md +265 -0
- package/upstream/benny/skills/triage-issue-reports/SKILL.md +240 -0
- package/upstream/benny/skills/triage-issue-reports/references/routing.example.md +61 -0
- package/upstream/benny/templates/configuration.example.yaml +84 -0
- package/upstream/benny/templates/reproduce-automation-prompt.md +33 -0
- package/upstream/benny/templates/triage-automation-prompt.md +39 -0
- package/upstream/mapping.json +699 -0
- package/upstream/poteto-mode/SKILL.md +110 -0
- package/upstream/poteto-mode/playbooks/authoring-a-skill.md +12 -0
- package/upstream/poteto-mode/playbooks/autonomous-run.md +13 -0
- package/upstream/poteto-mode/playbooks/autopilot-full.md +25 -0
- package/upstream/poteto-mode/playbooks/autopilot-stack.md +8 -0
- package/upstream/poteto-mode/playbooks/babysit.md +10 -0
- package/upstream/poteto-mode/playbooks/bug-fix.md +15 -0
- package/upstream/poteto-mode/playbooks/eval.md +25 -0
- package/upstream/poteto-mode/playbooks/feature.md +21 -0
- package/upstream/poteto-mode/playbooks/hillclimb.md +21 -0
- package/upstream/poteto-mode/playbooks/investigation.md +14 -0
- package/upstream/poteto-mode/playbooks/multi-phase-plan.md +11 -0
- package/upstream/poteto-mode/playbooks/opening-a-pr.md +9 -0
- package/upstream/poteto-mode/playbooks/pause-safely.md +10 -0
- package/upstream/poteto-mode/playbooks/perf-issue.md +25 -0
- package/upstream/poteto-mode/playbooks/prototype.md +14 -0
- package/upstream/poteto-mode/playbooks/refactoring.md +16 -0
- package/upstream/poteto-mode/playbooks/runtime-forensics.md +11 -0
- package/upstream/poteto-mode/playbooks/session-pickup.md +11 -0
- package/upstream/poteto-mode/playbooks/shipping.md +12 -0
- package/upstream/poteto-mode/playbooks/trace-forensics.md +14 -0
- package/upstream/poteto-mode/playbooks/visual-parity.md +11 -0
- package/upstream/poteto-mode/playbooks/worktree-cleanup.md +12 -0
- package/upstream/poteto-mode/references/bugbot-triage.md +142 -0
- package/upstream/poteto-mode/worker.md +11 -0
package/test/support.ts
ADDED
|
@@ -0,0 +1,541 @@
|
|
|
1
|
+
// What the pstack tests share: a host laid out as an installation (the packages pstack runs with in a
|
|
2
|
+
// `node_modules` above the home, where a stored script finds them), a real git repository, a fake GitLab
|
|
3
|
+
// server behind the forge module, a `proc.run` that runs where it is asked, and fake harnesses that answer each
|
|
4
|
+
// step of a playbook as the test says.
|
|
5
|
+
|
|
6
|
+
import { chmodSync, mkdirSync, readFileSync, realpathSync, symlinkSync, writeFileSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
|
|
9
|
+
import { z } from "zod";
|
|
10
|
+
|
|
11
|
+
import { action, actionRef, grant, principalOf } from "@niwa-labs/core";
|
|
12
|
+
import type { Caller } from "@niwa-labs/core";
|
|
13
|
+
import { evidence } from "@niwa-labs/evidence";
|
|
14
|
+
import { forge, secret } from "@niwa-labs/forge";
|
|
15
|
+
import { plan } from "@niwa-labs/plan";
|
|
16
|
+
import { AGENT, BUILTIN_WORKFLOW } from "@niwa-labs/plugin";
|
|
17
|
+
import type { AgentOffer, BuiltinWorkflow, CallerBinding, Plugin } from "@niwa-labs/plugin";
|
|
18
|
+
import { harnessList, sessionAsk, sessionEnd, sessionStart } from "@niwa-labs/sessions/records";
|
|
19
|
+
import { connect, connectAsOwner, createNiwa, eventually, git, plugin, requests, tempDir, timers, workflow } from "@niwa-labs/testing";
|
|
20
|
+
import type { Host } from "@niwa-labs/testing";
|
|
21
|
+
|
|
22
|
+
import { FakeGitLab, sha } from "../node_modules/@niwa-labs/forge/test/fake-gitlab.ts";
|
|
23
|
+
import type { FakeProject } from "../node_modules/@niwa-labs/forge/test/fake-gitlab.ts";
|
|
24
|
+
import { addressingOf } from "../src/binding.ts";
|
|
25
|
+
import type { Binding } from "../src/binding.ts";
|
|
26
|
+
import { pstackPlugin } from "../src/plugin.ts";
|
|
27
|
+
import type { RolesInput } from "../src/roles.ts";
|
|
28
|
+
import { fakePorts } from "./ports.ts";
|
|
29
|
+
import type { FakePorts } from "./ports.ts";
|
|
30
|
+
|
|
31
|
+
process.env.GIT_CONFIG_GLOBAL = "/dev/null";
|
|
32
|
+
process.env.GIT_CONFIG_NOSYSTEM = "1";
|
|
33
|
+
process.env.GIT_AUTHOR_NAME = "pstack test";
|
|
34
|
+
process.env.GIT_AUTHOR_EMAIL = "test@pstack.invalid";
|
|
35
|
+
process.env.GIT_COMMITTER_NAME = "pstack test";
|
|
36
|
+
process.env.GIT_COMMITTER_EMAIL = "test@pstack.invalid";
|
|
37
|
+
|
|
38
|
+
export const FORGE_HOST = "gitlab.example.com";
|
|
39
|
+
export const PROJECT = "acme/billing";
|
|
40
|
+
|
|
41
|
+
/** The roles of the tests: one fake harness each. */
|
|
42
|
+
export const ROLES: RolesInput = {
|
|
43
|
+
lead: { harness: "lead" },
|
|
44
|
+
author: { harness: "author", box: true },
|
|
45
|
+
reviewer: { harness: "reviewer" },
|
|
46
|
+
verifier: { harness: "verifier", vendor: "other" },
|
|
47
|
+
judge: { harness: "judge", vendor: "third" },
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
/** Run git in a directory; its output, trimmed. */
|
|
51
|
+
export function gitIn(dir: string, ...args: string[]): string {
|
|
52
|
+
const result = Bun.spawnSync(["git", "-C", dir, ...args], { stdout: "pipe", stderr: "pipe" });
|
|
53
|
+
if (result.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString()}`);
|
|
54
|
+
return result.stdout.toString().trim();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Write a file in a checkout and commit it: the new commit. */
|
|
58
|
+
export function commitFile(dir: string, file: string, text: string): string {
|
|
59
|
+
writeFileSync(join(dir, file), text);
|
|
60
|
+
gitIn(dir, "add", file);
|
|
61
|
+
gitIn(dir, "commit", "-q", "-m", `change ${file}`);
|
|
62
|
+
return gitIn(dir, "rev-parse", "HEAD");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
type VisibleActionRules = NonNullable<z.output<typeof sessionStart.input>["visible"]>["actions"];
|
|
66
|
+
|
|
67
|
+
/** What a fake harness is asked: by which harness, in which step, where, and what the session may call. */
|
|
68
|
+
export interface HarnessCall {
|
|
69
|
+
harness: string;
|
|
70
|
+
/** The step of the prompt's tag, `<playbook>:<step>`; empty without a tag. */
|
|
71
|
+
step: string;
|
|
72
|
+
prompt: string;
|
|
73
|
+
cwd?: string;
|
|
74
|
+
visible?: VisibleActionRules;
|
|
75
|
+
/** The reference of the conversation answering this turn. */
|
|
76
|
+
ref?: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** How the fakes answer a step: a value, or a function of the call. */
|
|
80
|
+
export type Answers = Record<string, (call: HarnessCall) => unknown>;
|
|
81
|
+
|
|
82
|
+
/** The step a prompt's tag names. */
|
|
83
|
+
const stepOf = (prompt: string): string => /\[pstack:([^\]]+)\]/.exec(prompt)?.[1] ?? "";
|
|
84
|
+
|
|
85
|
+
/** Fake calls and sessions seen through the workflow SDK. */
|
|
86
|
+
class Harnesses {
|
|
87
|
+
readonly calls: HarnessCall[] = [];
|
|
88
|
+
readonly sessions = new Map<string, {
|
|
89
|
+
id: string;
|
|
90
|
+
harness: string;
|
|
91
|
+
ended: boolean;
|
|
92
|
+
cwd?: string;
|
|
93
|
+
visible?: VisibleActionRules;
|
|
94
|
+
box: z.output<typeof sessionStart.input>["box"];
|
|
95
|
+
caller: Caller;
|
|
96
|
+
}>();
|
|
97
|
+
answers: Answers = {};
|
|
98
|
+
|
|
99
|
+
open(harness: string, caller: Caller, options: { cwd?: string; visible?: VisibleActionRules; box?: z.output<typeof sessionStart.input>["box"] }) {
|
|
100
|
+
const number = this.sessions.size + 1;
|
|
101
|
+
const id = `s-${String(number).padStart(6, "0")}`;
|
|
102
|
+
const ref = `${harness} native-${String(number)}`;
|
|
103
|
+
const { cwd, visible, box } = options;
|
|
104
|
+
this.sessions.set(ref, { id, harness, ended: false, ...(cwd === undefined ? {} : { cwd }), ...(visible === undefined ? {} : { visible }), box, caller });
|
|
105
|
+
return { session: id, ref, operation: `op-${String(number)}` };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
lookup(id: string | { harness: string; native: string }) {
|
|
109
|
+
const key = typeof id === "string" ? id : `${id.harness} ${id.native}`;
|
|
110
|
+
const entry = [...this.sessions].find(([ref, session]) => ref === key || session.id === key);
|
|
111
|
+
if (entry === undefined) throw new Error(`no fake session ${id}`);
|
|
112
|
+
return entry;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
answer(call: HarnessCall): unknown | Promise<unknown> {
|
|
116
|
+
this.calls.push(call);
|
|
117
|
+
const reply = this.answers[call.step] ?? (call.step === "" ? undefined : this.answers[`*:${call.step.slice(call.step.indexOf(":") + 1)}`]);
|
|
118
|
+
if (reply === undefined) throw new Error(`no fake answer for the step "${call.step}" (${call.harness})`);
|
|
119
|
+
return reply(call);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The calls of a step. */
|
|
123
|
+
of(step: string): HarnessCall[] {
|
|
124
|
+
return this.calls.filter((one) => one.step === step);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface PstackHost {
|
|
129
|
+
host: Host;
|
|
130
|
+
home: string;
|
|
131
|
+
repo: string;
|
|
132
|
+
worktrees: string;
|
|
133
|
+
gitlab: FakeGitLab;
|
|
134
|
+
project: FakeProject;
|
|
135
|
+
harnesses: Harnesses;
|
|
136
|
+
/** The agents the plugins offered to the agents module, as a stand-in for it read them. */
|
|
137
|
+
agents: AgentOffer[];
|
|
138
|
+
/** How the pipelines of the next pull requests end; a test may change it as it goes. */
|
|
139
|
+
ci: { head: string; mergeResult: string };
|
|
140
|
+
/** The fake providers of Benny's ports, and what they were asked. */
|
|
141
|
+
ports: FakePorts;
|
|
142
|
+
/** The fixture a run of the suite or the entry is given. */
|
|
143
|
+
fixture: { repo: { host: string; path: string; local: string }; worktrees: string; base: string };
|
|
144
|
+
/** Call an action as the owner, from the owner's own client. */
|
|
145
|
+
asOwner(name: string, input: unknown): Promise<unknown>;
|
|
146
|
+
/** Answer the owner's next open question with these option ids. */
|
|
147
|
+
answer(ids: string[]): Promise<void>;
|
|
148
|
+
/** Attest as the session whose mark is `mark`, as its own client: what a verifier does with `evidence.attest`. */
|
|
149
|
+
attestAs(mark: string, subject: string, claim: Record<string, unknown>): Promise<void>;
|
|
150
|
+
close(): Promise<void>;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** A `proc.run` that runs the command in its cwd, with no box: its target is the cwd, as the real action's is. */
|
|
154
|
+
function fakeProc(): Plugin {
|
|
155
|
+
const ref = actionRef("proc.run", {
|
|
156
|
+
description: "Run a command in its cwd.",
|
|
157
|
+
input: z.looseObject({ cmd: z.array(z.string()).min(1), cwd: z.string().min(1) }),
|
|
158
|
+
output: z.any(),
|
|
159
|
+
target: (input) => input.cwd,
|
|
160
|
+
});
|
|
161
|
+
return plugin("fake-proc", (api) =>
|
|
162
|
+
api.action(
|
|
163
|
+
action(ref, {
|
|
164
|
+
recovery: "repeat",
|
|
165
|
+
effect: "write",
|
|
166
|
+
world: "local",
|
|
167
|
+
run: ({ cmd, cwd }) => {
|
|
168
|
+
const ran = Bun.spawnSync(cmd, { cwd, stdout: "pipe", stderr: "pipe" });
|
|
169
|
+
const tail = (ran.stdout.toString() + ran.stderr.toString()).slice(-4096);
|
|
170
|
+
return {
|
|
171
|
+
outcome: "exited",
|
|
172
|
+
exit: ran.exitCode,
|
|
173
|
+
durationMs: 1,
|
|
174
|
+
tail,
|
|
175
|
+
truncated: false,
|
|
176
|
+
output: { digest: `sha256:${"0".repeat(64)}`, size: tail.length, type: "text/plain" },
|
|
177
|
+
box: { sandbox: "none", write: [cwd], net: [] },
|
|
178
|
+
};
|
|
179
|
+
},
|
|
180
|
+
}),
|
|
181
|
+
),
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Sessions started and asked through `session.start` and `session.ask`, answered by the fakes. Like the sessions
|
|
187
|
+
* module, it says which session a token's mark names, so a session that attests is bound to its address; the mark
|
|
188
|
+
* of a fake session is its native id.
|
|
189
|
+
*/
|
|
190
|
+
function fakeSessions(harnesses: Harnesses): Binding {
|
|
191
|
+
const made_ = plugin("sessions", (api) => {
|
|
192
|
+
api.action(action(harnessList, {
|
|
193
|
+
recovery: "repeat",
|
|
194
|
+
effect: "read",
|
|
195
|
+
world: "niwa",
|
|
196
|
+
run: () => ({ items: [] }),
|
|
197
|
+
}));
|
|
198
|
+
api.action(
|
|
199
|
+
action("session.start", {
|
|
200
|
+
description: "Start a fake session.",
|
|
201
|
+
input: sessionStart.input,
|
|
202
|
+
output: z.any(),
|
|
203
|
+
recovery: "repeat",
|
|
204
|
+
effect: "write",
|
|
205
|
+
world: "local",
|
|
206
|
+
run: (input, { caller }) => {
|
|
207
|
+
const { cwd, box } = input;
|
|
208
|
+
const visible = input.visible?.actions;
|
|
209
|
+
return harnesses.open(input.harness, caller, {
|
|
210
|
+
...(cwd === undefined ? {} : { cwd }),
|
|
211
|
+
...(visible === undefined ? {} : { visible }),
|
|
212
|
+
...(box === undefined ? {} : { box }),
|
|
213
|
+
});
|
|
214
|
+
},
|
|
215
|
+
}),
|
|
216
|
+
);
|
|
217
|
+
api.action(action(sessionAsk, {
|
|
218
|
+
recovery: "repeat",
|
|
219
|
+
effect: "write",
|
|
220
|
+
world: "local",
|
|
221
|
+
async run(input) {
|
|
222
|
+
const [ref, session] = harnesses.lookup(input.session);
|
|
223
|
+
if (session.ended) throw new Error(`fake session ${ref} already ended`);
|
|
224
|
+
const prompt = input.prompt.sections.map((section) => section.text).join("\n");
|
|
225
|
+
const value = await harnesses.answer({ harness: session.harness, step: stepOf(prompt), prompt, ref, ...(session.cwd === undefined ? {} : { cwd: session.cwd }), ...(session.visible === undefined ? {} : { visible: session.visible }) });
|
|
226
|
+
return { value };
|
|
227
|
+
},
|
|
228
|
+
}));
|
|
229
|
+
api.action(action(sessionEnd, {
|
|
230
|
+
recovery: "repeat",
|
|
231
|
+
effect: "write",
|
|
232
|
+
world: "local",
|
|
233
|
+
run: ({ session: id }) => {
|
|
234
|
+
const [, session] = harnesses.lookup(id);
|
|
235
|
+
const ended = !session.ended;
|
|
236
|
+
session.ended = true;
|
|
237
|
+
return { ended };
|
|
238
|
+
},
|
|
239
|
+
}));
|
|
240
|
+
});
|
|
241
|
+
return Object.assign(made_, {
|
|
242
|
+
bind: (fallback?: CallerBinding): CallerBinding => (caller) => {
|
|
243
|
+
const principal = caller.kind === "client" ? caller.principal : undefined;
|
|
244
|
+
const found = principal?.kind === "session" ? [...harnesses.sessions.entries()].find(([ref]) => ref.endsWith(` ${principal.id}`)) : undefined;
|
|
245
|
+
if (found === undefined || principal === undefined) return fallback?.(caller);
|
|
246
|
+
return { kind: "session", harness: found[1].harness, id: principal.id };
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** The workflow SDK action of each role's harness. */
|
|
252
|
+
function fakeAgents(harnesses: Harnesses) {
|
|
253
|
+
const names = ["lead", "author", "reviewer", "verifier", "judge"];
|
|
254
|
+
return Object.fromEntries(
|
|
255
|
+
names.map((harness) => [
|
|
256
|
+
harness,
|
|
257
|
+
action(`session.start.${harness}`, {
|
|
258
|
+
description: "Open a fake workflow session or answer a one-shot question.",
|
|
259
|
+
input: z.looseObject({ prompt: z.string().optional(), cwd: z.string().optional(), visible: z.array(z.string()).optional(), box: z.custom<z.output<typeof sessionStart.input>["box"]>().optional() }),
|
|
260
|
+
output: z.any(),
|
|
261
|
+
recovery: "repeat",
|
|
262
|
+
effect: "write",
|
|
263
|
+
world: "local",
|
|
264
|
+
async run({ prompt, cwd, visible, box }, { caller }) {
|
|
265
|
+
const options = { ...(cwd === undefined ? {} : { cwd }), ...(visible === undefined ? {} : { visible }) };
|
|
266
|
+
if (prompt === undefined) return harnesses.open(harness, caller, { ...options, ...(box === undefined ? {} : { box }) });
|
|
267
|
+
return await harnesses.answer({ harness, step: stepOf(prompt), prompt, ...options });
|
|
268
|
+
},
|
|
269
|
+
}),
|
|
270
|
+
]),
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* The fake server's fetch, with a CI: a pull request opened gets a pipeline of its head and one of its merge result,
|
|
276
|
+
* the merge of its head into the local trunk's tip, ending as `ci` says.
|
|
277
|
+
*/
|
|
278
|
+
function withCi(gitlab: FakeGitLab, project: FakeProject, repo: string, ci: { head: string; mergeResult: string }): FakeGitLab["fetch"] {
|
|
279
|
+
return async (url, init) => {
|
|
280
|
+
const answered = await gitlab.fetch(url, init);
|
|
281
|
+
if (init.method === "POST" && /\/merge_requests$/.test(new URL(url).pathname) && answered.status === 201) {
|
|
282
|
+
const mr = (await answered.clone().json()) as { iid: number; sha: string; target_branch: string };
|
|
283
|
+
const target = gitIn(repo, "rev-parse", mr.target_branch);
|
|
284
|
+
const merge = sha(`merge-result-${String(mr.iid)}-${mr.sha}`);
|
|
285
|
+
project.commits.set(merge, [target, mr.sha]);
|
|
286
|
+
gitlab.pipeline(project, { sha: mr.sha, ref: `refs/merge-requests/${String(mr.iid)}/head`, status: ci.head, source: "merge_request_event" });
|
|
287
|
+
gitlab.pipeline(project, { sha: merge, ref: `refs/merge-requests/${String(mr.iid)}/merge`, status: ci.mergeResult, source: "merge_request_event" });
|
|
288
|
+
}
|
|
289
|
+
return answered;
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** What a test host is made with. */
|
|
294
|
+
export interface PstackHostOptions {
|
|
295
|
+
/** Install only these real built-ins ungated, for isolated playbook tests. */
|
|
296
|
+
readonly onlyBuiltins?: readonly string[];
|
|
297
|
+
/** The binding evidence binds its callers with. Default: pstack's, over the fake sessions. */
|
|
298
|
+
readonly bind?: CallerBinding;
|
|
299
|
+
/** Benny: on unless `false`. */
|
|
300
|
+
readonly benny?: false;
|
|
301
|
+
/** The role seats, when a test needs specific harnesses or vendors. */
|
|
302
|
+
readonly roles?: RolesInput;
|
|
303
|
+
/** The providers of Benny's ports that are left out, as a host that lacks them would be. */
|
|
304
|
+
readonly withoutPorts?: ReadonlyArray<"tracker" | "channel" | "control">;
|
|
305
|
+
/** How the pipelines a new pull request gets end: its head's and its merge result's. Default: both succeed. */
|
|
306
|
+
readonly ci?: { readonly head?: string; readonly mergeResult?: string };
|
|
307
|
+
/** More plugins, installed before the workflow module. */
|
|
308
|
+
readonly plugins?: readonly Plugin[];
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* A host with pstack: requests, timers, git, a fake `proc.run`, fake sessions, the forge module over a fake GitLab
|
|
313
|
+
* with the project acme/billing (42), evidence, plan, pstack, and the workflow module with a fake harness action per role.
|
|
314
|
+
*/
|
|
315
|
+
export async function pstackHost(options: PstackHostOptions = {}): Promise<PstackHost> {
|
|
316
|
+
const root = realpathSync(await tempDir("pstack"));
|
|
317
|
+
// As in an installation, the packages pstack runs with are in a node_modules above the home.
|
|
318
|
+
symlinkSync(realpathSync(join(import.meta.dir, "..", "node_modules")), join(root, "node_modules"), "dir");
|
|
319
|
+
const home = join(root, "home");
|
|
320
|
+
mkdirSync(join(home, "secrets"), { recursive: true });
|
|
321
|
+
const repo = join(root, "repo");
|
|
322
|
+
mkdirSync(repo);
|
|
323
|
+
gitIn(repo, "init", "-q", "-b", "main");
|
|
324
|
+
writeFileSync(join(repo, "README.md"), "The billing service.\n");
|
|
325
|
+
gitIn(repo, "add", ".");
|
|
326
|
+
gitIn(repo, "commit", "-q", "-m", "start");
|
|
327
|
+
|
|
328
|
+
const gitlab = new FakeGitLab();
|
|
329
|
+
const project = gitlab.project(42, PROJECT);
|
|
330
|
+
project.branches.set("main", gitIn(repo, "rev-parse", "main"));
|
|
331
|
+
const token = join(home, "secrets", "GITLAB_TOKEN");
|
|
332
|
+
writeFileSync(token, `${gitlab.token}\n`);
|
|
333
|
+
chmodSync(token, 0o600);
|
|
334
|
+
|
|
335
|
+
const harnesses = new Harnesses();
|
|
336
|
+
const agents = fakeAgents(harnesses);
|
|
337
|
+
const sessions = fakeSessions(harnesses);
|
|
338
|
+
const offeredAgents: AgentOffer[] = [];
|
|
339
|
+
const ports = fakePorts(options.withoutPorts ?? []);
|
|
340
|
+
const ci = { head: options.ci?.head ?? "success", mergeResult: options.ci?.mergeResult ?? "success" };
|
|
341
|
+
const host = createNiwa({ home });
|
|
342
|
+
const pstack = pstackPlugin({ roles: options.roles ?? ROLES, forges: [FORGE_HOST], ...(options.benny === false ? {} : { benny: { repo: { host: FORGE_HOST, path: PROJECT, local: repo } } }) });
|
|
343
|
+
if (options.onlyBuiltins) {
|
|
344
|
+
const setup = pstack.setup;
|
|
345
|
+
pstack.setup = api => setup({ ...api, offer(key, value) {
|
|
346
|
+
if (key.name !== BUILTIN_WORKFLOW.name) return api.offer(key, value);
|
|
347
|
+
const builtin = value as BuiltinWorkflow;
|
|
348
|
+
if (options.onlyBuiltins?.includes(builtin.name)) api.offer(BUILTIN_WORKFLOW, { ...builtin, gated: false });
|
|
349
|
+
} });
|
|
350
|
+
}
|
|
351
|
+
host.use(
|
|
352
|
+
requests(),
|
|
353
|
+
timers([]),
|
|
354
|
+
git({ cleanup: false }),
|
|
355
|
+
fakeProc(),
|
|
356
|
+
plugin("box", (api) => {
|
|
357
|
+
api.action(action("box.net", {
|
|
358
|
+
description: "A rule allowing a box to reach a host.",
|
|
359
|
+
input: z.strictObject({ host: z.string().min(1) }),
|
|
360
|
+
output: z.strictObject({ held: z.literal(true) }),
|
|
361
|
+
recovery: "repeat",
|
|
362
|
+
effect: "read",
|
|
363
|
+
world: "local",
|
|
364
|
+
target: (input) => input.host,
|
|
365
|
+
run: () => ({ held: true as const }),
|
|
366
|
+
}));
|
|
367
|
+
}),
|
|
368
|
+
sessions,
|
|
369
|
+
plugin("fake-agents", (api) => {
|
|
370
|
+
for (const one of Object.values(agents)) api.action(one);
|
|
371
|
+
}),
|
|
372
|
+
forge({ servers: [{ host: FORGE_HOST, url: gitlab.url, token: secret("GITLAB_TOKEN") }], fetch: withCi(gitlab, project, repo, ci), env: {}, poll: false }),
|
|
373
|
+
evidence({ bind: options.bind ?? addressingOf([sessions]) }),
|
|
374
|
+
plan(),
|
|
375
|
+
...(options.plugins ?? []),
|
|
376
|
+
...(options.benny === false ? [] : ports.plugins),
|
|
377
|
+
pstack,
|
|
378
|
+
// The agents module reads the agents a plugin offers; this one only keeps them.
|
|
379
|
+
plugin("agents", (api) => {
|
|
380
|
+
offeredAgents.push(...api.offers(AGENT).map((one) => one.value));
|
|
381
|
+
}),
|
|
382
|
+
workflow({ agents }),
|
|
383
|
+
);
|
|
384
|
+
await host.serve();
|
|
385
|
+
const owner = await connectAsOwner(home);
|
|
386
|
+
const worktrees = join(home, "worktrees");
|
|
387
|
+
return {
|
|
388
|
+
host,
|
|
389
|
+
home,
|
|
390
|
+
repo,
|
|
391
|
+
worktrees,
|
|
392
|
+
gitlab,
|
|
393
|
+
project,
|
|
394
|
+
harnesses,
|
|
395
|
+
ports,
|
|
396
|
+
agents: offeredAgents,
|
|
397
|
+
ci,
|
|
398
|
+
fixture: { repo: { host: FORGE_HOST, path: PROJECT, local: repo }, worktrees, base: "main" },
|
|
399
|
+
answer: async (ids) => {
|
|
400
|
+
let id: string | undefined;
|
|
401
|
+
await eventually(async () => {
|
|
402
|
+
const open = (await owner.call("request.open" as never, {} as never)) as { items?: Array<{ id: string }>; requests?: Array<{ id: string }> };
|
|
403
|
+
id = (open.items ?? open.requests ?? [])[0]?.id;
|
|
404
|
+
return id !== undefined;
|
|
405
|
+
}, 20_000, "the owner was not asked");
|
|
406
|
+
await owner.call("request.answer" as never, { id, answer: { ids } } as never);
|
|
407
|
+
},
|
|
408
|
+
asOwner: async (name, input) => await owner.call(name as never, input as never),
|
|
409
|
+
attestAs: async (mark, subject, claim) => {
|
|
410
|
+
const token = host.issueToken(principalOf("session", mark), { grant: grant(["evidence.attest"]) });
|
|
411
|
+
const client = await connect({ home, id: `session:${mark}`, token });
|
|
412
|
+
try {
|
|
413
|
+
await client.call("evidence.attest" as never, { subject, claim } as never);
|
|
414
|
+
} finally {
|
|
415
|
+
await client.close().catch(() => {});
|
|
416
|
+
}
|
|
417
|
+
},
|
|
418
|
+
close: async () => {
|
|
419
|
+
await owner.close().catch(() => {});
|
|
420
|
+
await host.close().catch(() => {});
|
|
421
|
+
},
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** pstack's built-ins as the host lists them, by name. */
|
|
426
|
+
export async function builtins(host: Host): Promise<Record<string, { id: string; version: string }>> {
|
|
427
|
+
const listed = (await host.call("workflow.list" as never, {} as never)) as { items: Array<{ id: string; version: string; builtin?: { plugin: string; name: string } }> };
|
|
428
|
+
return Object.fromEntries(listed.items.filter((one) => one.builtin?.plugin === "pstack").map((one) => [one.builtin?.name ?? "", { id: one.id, version: one.version }]));
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** The records of a type. */
|
|
432
|
+
export async function recordsOf(host: Host, type: string): Promise<Array<{ data: unknown; seq: number }>> {
|
|
433
|
+
return (await host.records.read({ types: [type] })).map((row) => ({ data: row.data, seq: row.seq }));
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/** The node ids a fake lead plans: two independent pull requests, or a stack of two when the task asks for one. */
|
|
437
|
+
const PARALLEL = [
|
|
438
|
+
{ id: "changelog", text: "Add a changelog", after: [], files: ["CHANGELOG.md"] },
|
|
439
|
+
{ id: "contributing", text: "Add a contributing guide", after: [], files: ["CONTRIBUTING.md"] },
|
|
440
|
+
];
|
|
441
|
+
const STACK = [
|
|
442
|
+
{ id: "settings", text: "Add a settings module", after: [], files: ["settings.md"] },
|
|
443
|
+
{ id: "page", text: "Add a page that uses the settings", after: ["settings"], files: ["page.md"] },
|
|
444
|
+
];
|
|
445
|
+
|
|
446
|
+
/** The claim a lane asks a verifier to attest, as its prompt names it. */
|
|
447
|
+
export function claimAsked(prompt: string): { subject: string; name: string; sha: string } {
|
|
448
|
+
const found = /subject "([^"]+)", claim \{ kind: "verdict", name: "([^"]+)", outcome: "pass" or "fail", sha: "([0-9a-f]+)" \}/.exec(prompt);
|
|
449
|
+
if (found === null) throw new Error("the verifier was not told what to attest");
|
|
450
|
+
return { subject: found[1] as string, name: found[2] as string, sha: found[3] as string };
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Answers that carry every playbook through on the fakes: the lead routes to the investigation and plans, the author
|
|
455
|
+
* commits in its worktree and publishes the branch to the fake server, and a verifier attests a pass as itself.
|
|
456
|
+
*/
|
|
457
|
+
export function defaultAnswers(made: PstackHost): Answers {
|
|
458
|
+
let changes = 0;
|
|
459
|
+
/** What an author does: commit a change in its worktree, and publish the branch to the fake server. */
|
|
460
|
+
const commit = (call: HarnessCall): string => {
|
|
461
|
+
if (call.cwd === undefined) throw new Error("the author was started in no directory");
|
|
462
|
+
changes += 1;
|
|
463
|
+
const sha = commitFile(call.cwd, `change-${String(changes)}.md`, `change ${String(changes)}\n`);
|
|
464
|
+
made.project.branches.set(gitIn(call.cwd, "rev-parse", "--abbrev-ref", "HEAD"), sha);
|
|
465
|
+
return sha;
|
|
466
|
+
};
|
|
467
|
+
const verify = (evidence: string) => async (call: HarnessCall) => {
|
|
468
|
+
const asked = claimAsked(call.prompt);
|
|
469
|
+
await made.attestAs(call.ref?.split(" ")[1] ?? "", asked.subject, { kind: "verdict", name: asked.name, outcome: "pass", sha: asked.sha });
|
|
470
|
+
return { outcome: "pass", evidence };
|
|
471
|
+
};
|
|
472
|
+
const hypotheses = () => ({ cause: "The README was not updated.", hypotheses: [{ claim: "The README was not updated", status: "confirmed", evidence: "README.md:1" }] });
|
|
473
|
+
return {
|
|
474
|
+
"entry:route": () => ({ kind: "run", playbook: "investigation", task: "Where does this repository describe itself?", more: {} }),
|
|
475
|
+
"investigation:trace": () => ({ summary: "The README says what the service is.", sources: [{ file: "README.md", line: 1 }] }),
|
|
476
|
+
"multi-phase-plan:plan": (call) => ({ kind: "plan", goal: "Deliver the task", nodes: /stack|building on|builds on/i.test(call.prompt.slice(call.prompt.lastIndexOf("Task:"))) ? STACK : PARALLEL }),
|
|
477
|
+
"*:build": (call) => ({ sha: commit(call), summary: `Added change ${String(changes)}.` }),
|
|
478
|
+
"*:verify-gates": verify("the gates passed"),
|
|
479
|
+
"*:verify-live": verify("the page renders"),
|
|
480
|
+
"trace-forensics:investigate": hypotheses,
|
|
481
|
+
"runtime-forensics:investigate": hypotheses,
|
|
482
|
+
"feature:frame": () => ({ shape: "A section of the README.", checkpoint: { blocking: "n/a: one file", independent: "n/a: one file", shared: "n/a: none", decomposition: "one worker: one file" } }),
|
|
483
|
+
"bug-fix:frame": () => ({ repro: "read README.md", cause: "a typo", evidence: "README.md:1" }),
|
|
484
|
+
"refactoring:frame": () => ({ contract: "The README reads the same.", pin: "test -f README.md", target: "One section per topic." }),
|
|
485
|
+
"authoring-a-skill:frame": () => ({ skill: "readme-style", decisions: ["short"], validation: "frontmatter has a name" }),
|
|
486
|
+
"visual-parity:frame": () => ({ reference: "the design", surface: "the README", differences: ["the heading"] }),
|
|
487
|
+
"perf-issue:frame": () => ({ measure: "ls | wc -l", direction: "higher", target: 1 }),
|
|
488
|
+
"autonomous-run:attempt": (call) => ({ change: `committed ${commit(call)}` }),
|
|
489
|
+
"hillclimb:attempt": (call) => ({ hypothesis: `one more note: ${commit(call)}` }),
|
|
490
|
+
"prototype:scope": () => ({ kind: "decide", decision: "What the README leads with", variants: [{ name: "usage", idea: "Usage first" }, { name: "architecture", idea: "Architecture first" }] }),
|
|
491
|
+
"prototype:build": (call) => ({ observed: `built in ${call.cwd ?? ""}` }),
|
|
492
|
+
"prototype:recommend": () => ({ recommend: "usage", why: "A reader wants to run it first." }),
|
|
493
|
+
"eval:frame": () => ({
|
|
494
|
+
success: "The explanation is right and short.",
|
|
495
|
+
rubric: [
|
|
496
|
+
{ id: "right", text: "It is right." },
|
|
497
|
+
{ id: "short", text: "It is short." },
|
|
498
|
+
{ id: "plain", text: "It is plain." },
|
|
499
|
+
],
|
|
500
|
+
cases: [{ id: "readme", prompt: "What is this repository for?" }],
|
|
501
|
+
}),
|
|
502
|
+
// Prompts with no tag: an eval's candidates, which must not see they are graded, and its grader.
|
|
503
|
+
"": (call) => {
|
|
504
|
+
if (!call.prompt.startsWith("Grade the subject")) return { answer: "It is the billing service." };
|
|
505
|
+
const ids = [...call.prompt.matchAll(/^- ([a-z][a-z0-9-]*): /gm)].map((found) => found[1] as string);
|
|
506
|
+
return { scores: ids.map((id) => ({ id, score: 1 })) };
|
|
507
|
+
},
|
|
508
|
+
"opening-a-pr:write": () => ({ title: "docs(readme): add the page", sections: [{ heading: "Why", body: "The page needs it." }, { heading: "Verification", body: "Read it." }] }),
|
|
509
|
+
"pause-safely:checkpoint": (call) => {
|
|
510
|
+
commit(call);
|
|
511
|
+
return { intent: "Add the page", state: "half done", verified: [], next: "finish the page", files: ["page.md"], gotchas: [] };
|
|
512
|
+
},
|
|
513
|
+
"benny-triage:triage": () => ({ verdict: "bug", owner: "the README", summary: "The README names the wrong service." }),
|
|
514
|
+
"benny-reproduce:reproduce": () => ({ reproduced: true, runs: ["The README says payroll.", "It still says payroll."] }),
|
|
515
|
+
// What a boxed author does for the sync: it reads the report in its worktree and settles the touched entries as carried.
|
|
516
|
+
"upstream-sync:round": (call) => {
|
|
517
|
+
const report = JSON.parse(readFileSync(join(call.cwd as string, ".upstream-map-report.json"), "utf8")) as { entries: Array<{ id: string; status: string }> };
|
|
518
|
+
return { summary: "settled the touched entries", decisions: [], accept: report.entries.filter((one) => one.status === "touched").map((one) => one.id) };
|
|
519
|
+
},
|
|
520
|
+
"session-pickup:resume": () => ({ where: "Both pull requests merged.", verified: ["changelog", "contributing"], next: "Nothing is left." }),
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/** The steps that failed, by the prefix of their key: the workflow module's sync by default. */
|
|
525
|
+
export async function syncErrors(host: Host, prefix = "sync:"): Promise<string[]> {
|
|
526
|
+
return (await host.records.read({ types: ["niwa.step.outcome"] }))
|
|
527
|
+
.map((row) => row.data as { key: string; outcome: { status: string; error?: string } })
|
|
528
|
+
.filter((one) => one.key.startsWith(prefix) && one.outcome.status === "failed")
|
|
529
|
+
.map((one) => `${one.key}: ${one.outcome.error ?? ""}`);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** Wait until every built-in of pstack is registered, or fail with why the sync refused one. */
|
|
533
|
+
export async function registeredAll(host: Host, count: number): Promise<void> {
|
|
534
|
+
const deadline = Date.now() + 30_000;
|
|
535
|
+
while (Object.keys(await builtins(host)).length < count) {
|
|
536
|
+
const errors = await syncErrors(host);
|
|
537
|
+
if (errors.length > 0) throw new Error(`pstack's built-ins were not registered:\n${errors.join("\n")}`);
|
|
538
|
+
if (Date.now() > deadline) throw new Error("pstack's built-ins were not registered in time");
|
|
539
|
+
await Bun.sleep(100);
|
|
540
|
+
}
|
|
541
|
+
}
|