@niwa-labs/workflow 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +13 -2
- package/package.json +55 -4
- package/src/actions.ts +408 -0
- package/src/agent.ts +211 -0
- package/src/ask.ts +84 -0
- package/src/bridge.ts +710 -0
- package/src/catalogue.ts +74 -0
- package/src/check.ts +880 -0
- package/src/child.ts +1378 -0
- package/src/cockpit/actions.ts +128 -0
- package/src/cockpit/catalogue-actions.ts +97 -0
- package/src/cockpit/dashboard.tsx +134 -0
- package/src/cockpit/groups.ts +45 -0
- package/src/cockpit/index.tsx +50 -0
- package/src/cockpit/launch.tsx +136 -0
- package/src/cockpit/plan.ts +101 -0
- package/src/cockpit/run-commands.tsx +102 -0
- package/src/cockpit/run-page.tsx +140 -0
- package/src/cockpit/run-summary.ts +16 -0
- package/src/cockpit/runs.tsx +73 -0
- package/src/cockpit/statistics.tsx +176 -0
- package/src/cockpit/track.tsx +159 -0
- package/src/cockpit/workflows.tsx +383 -0
- package/src/describe-cache.ts +122 -0
- package/src/describe.ts +174 -0
- package/src/guide.ts +213 -0
- package/src/index.ts +2813 -0
- package/src/inspect.ts +235 -0
- package/src/installation.ts +90 -0
- package/src/language.ts +217 -0
- package/src/records-entry.ts +63 -0
- package/src/records.ts +574 -0
- package/src/registry.ts +250 -0
- package/src/revise.ts +89 -0
- package/src/runner.ts +339 -0
- package/src/script-types.ts +549 -0
- package/src/stats.ts +460 -0
- package/src/status.ts +512 -0
- package/src/store.ts +259 -0
- package/src/subscriptions.ts +102 -0
- package/src/waiting.ts +61 -0
- package/test/cockpit-actions.test.ts +20 -0
- package/test/cockpit-groups.test.ts +42 -0
- package/test/cockpit-plan.test.ts +118 -0
- package/test/fixtures/branches-host.ts +58 -0
- package/test/fixtures/branches-script.js +23 -0
- package/test/fixtures/bridge-host.ts +23 -0
- package/test/fixtures/bridge-keys-script.js +15 -0
- package/test/fixtures/bridge-wait-script.js +8 -0
- package/test/fixtures/cancel-host.ts +58 -0
- package/test/fixtures/cancel-script.js +12 -0
- package/test/fixtures/crash-host.ts +61 -0
- package/test/fixtures/crash-script.js +14 -0
- package/test/fixtures/ping-records.ts +20 -0
- package/test/fixtures/revision-host.ts +57 -0
- package/test/fixtures/revision-script.js +11 -0
- package/test/fixtures/running-host.ts +61 -0
- package/test/fixtures/running-script.js +12 -0
- package/test/fixtures/with-data/flow.ts +2 -0
- package/test/fixtures/with-data/table.json +1 -0
- package/test/git-repo.ts +45 -0
- package/test/git-worktree.test.ts +413 -0
- package/test/question-format.test.ts +279 -0
- package/test/workflow-agent-caps.test.ts +91 -0
- package/test/workflow-agent-lifetime.test.ts +93 -0
- package/test/workflow-agent-options.test.ts +216 -0
- package/test/workflow-agent-schema.test.ts +386 -0
- package/test/workflow-agent-typings.test.ts +241 -0
- package/test/workflow-ask-resume.test.ts +145 -0
- package/test/workflow-branch-keys.test.ts +133 -0
- package/test/workflow-bridge-inproc.test.ts +230 -0
- package/test/workflow-bridge.test.ts +141 -0
- package/test/workflow-builtin-candidate.test.ts +230 -0
- package/test/workflow-builtin.test.ts +197 -0
- package/test/workflow-cancel-agent.test.ts +144 -0
- package/test/workflow-cancel.test.ts +166 -0
- package/test/workflow-chain.test.ts +84 -0
- package/test/workflow-check-options.test.ts +144 -0
- package/test/workflow-check-uses.test.ts +148 -0
- package/test/workflow-check.test.ts +282 -0
- package/test/workflow-contract.test.ts +295 -0
- package/test/workflow-crash-resume.test.ts +141 -0
- package/test/workflow-deadline.test.ts +102 -0
- package/test/workflow-describe-cache.test.ts +240 -0
- package/test/workflow-directory.test.ts +73 -0
- package/test/workflow-emits.test.ts +261 -0
- package/test/workflow-group-keys.test.ts +72 -0
- package/test/workflow-harnesses.test.ts +262 -0
- package/test/workflow-inspect.test.ts +152 -0
- package/test/workflow-language-types.test.ts +148 -0
- package/test/workflow-manifest.test.ts +93 -0
- package/test/workflow-manual-pause.test.ts +116 -0
- package/test/workflow-owners.test.ts +154 -0
- package/test/workflow-pause.test.ts +200 -0
- package/test/workflow-phases.test.ts +445 -0
- package/test/workflow-plugin-entries.test.ts +59 -0
- package/test/workflow-record-refs.test.ts +310 -0
- package/test/workflow-replay.test.ts +206 -0
- package/test/workflow-resume-once.test.ts +188 -0
- package/test/workflow-revision.test.ts +216 -0
- package/test/workflow-scope.test.ts +505 -0
- package/test/workflow-script-defaults.test.ts +119 -0
- package/test/workflow-script-error.test.ts +228 -0
- package/test/workflow-script-libraries.test.ts +88 -0
- package/test/workflow-script-packages.test.ts +119 -0
- package/test/workflow-stdout.test.ts +89 -0
- package/test/workflow-stop.test.ts +299 -0
- package/test/workflow-store.test.ts +220 -0
- package/test/workflow-visible.test.ts +136 -0
- package/test/workflow-wait-version.test.ts +54 -0
- package/test/workflow-wait.test.ts +224 -0
- package/test/workflow-waiting-agent.test.ts +64 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
// An ask() call may carry the options block of its harness's kind: the block crosses the script's
|
|
2
|
+
// process, the request to the host, the agent action and session.start, and arrives at the control
|
|
3
|
+
// side. A block of another kind, or a field the block does not have, fails the call before a session
|
|
4
|
+
// starts: the agent action of the harness takes the block of the kind the sessions module binds it to,
|
|
5
|
+
// and no other. The run's own check (workflow.check) knows each harness's block from the same place.
|
|
6
|
+
|
|
7
|
+
import { expect, test } from "bun:test";
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
|
|
10
|
+
import { rmSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
|
|
13
|
+
import { action } from "@niwa-labs/core";
|
|
14
|
+
import type { ActionName } from "@niwa-labs/core";
|
|
15
|
+
import type { Plugin } from "@niwa-labs/plugin";
|
|
16
|
+
import { harnessId, kindOptions, machineId, sessionId } from "@niwa-labs/harness";
|
|
17
|
+
import type { Harness, KindOptions, SessionControl, SessionId, StartOptions, StartedSession } from "@niwa-labs/harness";
|
|
18
|
+
import { createNiwa, plugin, sessions, tempDirSync, watch } from "@niwa-labs/testing";
|
|
19
|
+
import { workflow } from "../src/index.ts";
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
const HARNESS = harnessId("claude-sonnet");
|
|
23
|
+
/** The block of the Claude kinds, as their package declares it. */
|
|
24
|
+
const CLAUDE = kindOptions("claude", z.strictObject({ advisor: z.boolean().optional() }));
|
|
25
|
+
|
|
26
|
+
/** A control side that keeps what each session was started with and answers every question. */
|
|
27
|
+
class RecordingControl implements SessionControl {
|
|
28
|
+
readonly starts: StartOptions[] = [];
|
|
29
|
+
|
|
30
|
+
async start(options: StartOptions): Promise<StartedSession> {
|
|
31
|
+
this.starts.push(options);
|
|
32
|
+
return { session: sessionId(HARNESS, `session-${String(this.starts.length)}`) };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async resume(): Promise<void> {
|
|
36
|
+
throw new Error("this session cannot be resumed");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async send() {
|
|
40
|
+
return { status: "started" as const, turn: 1 };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async interrupt(): Promise<void> {}
|
|
44
|
+
|
|
45
|
+
async ask<T>(id: SessionId): Promise<{ session: SessionId; value: T; at: never }> {
|
|
46
|
+
return { session: id, value: { answer: "ok" } as T, at: 0 as never };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async *life(): AsyncIterable<never> {}
|
|
50
|
+
|
|
51
|
+
async state(): Promise<"unknown"> {
|
|
52
|
+
return "unknown";
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const instance: Harness = {
|
|
57
|
+
id: HARNESS,
|
|
58
|
+
kind: "claude-code" as never,
|
|
59
|
+
machine: machineId("machine-1"),
|
|
60
|
+
home: "/tmp/fake-home",
|
|
61
|
+
sides: {
|
|
62
|
+
control: {
|
|
63
|
+
start: true,
|
|
64
|
+
resume: false,
|
|
65
|
+
send: true,
|
|
66
|
+
interrupt: true,
|
|
67
|
+
ask: true,
|
|
68
|
+
structuredOutput: true,
|
|
69
|
+
launch: { model: false, effort: [], modes: [] },
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/** Runs a script on a host over the recording control side, until `done` holds of what the journal has seen. */
|
|
75
|
+
async function run(
|
|
76
|
+
script: string,
|
|
77
|
+
done: (seen: string) => boolean,
|
|
78
|
+
binding: { options?: KindOptions } = { options: CLAUDE },
|
|
79
|
+
extra: readonly Plugin[] = [],
|
|
80
|
+
): Promise<{ control: RecordingControl; seen: string }> {
|
|
81
|
+
const control = new RecordingControl();
|
|
82
|
+
const home = tempDirSync("agent-options");
|
|
83
|
+
const niwa = createNiwa({ home });
|
|
84
|
+
niwa.use(sessions({ harnesses: [{ instance, control, ...binding }] }), ...extra);
|
|
85
|
+
niwa.use(workflow({ harnesses: [HARNESS] }));
|
|
86
|
+
await niwa.serve();
|
|
87
|
+
try {
|
|
88
|
+
const file = join(home, "script.js");
|
|
89
|
+
await Bun.write(file, script);
|
|
90
|
+
await niwa.call("workflow.start" as ActionName, { file } as never);
|
|
91
|
+
const observer = await watch(home);
|
|
92
|
+
for (let waited = 0; waited < 10_000 && !done(JSON.stringify(observer.seen)); waited += 100) await Bun.sleep(100);
|
|
93
|
+
await observer.stop();
|
|
94
|
+
return { control, seen: JSON.stringify(observer.seen) };
|
|
95
|
+
} finally {
|
|
96
|
+
await niwa.close();
|
|
97
|
+
rmSync(home, { recursive: true, force: true });
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
test("the block of the harness's kind in an ask() call reaches the control side; a call without one carries none", async () => {
|
|
102
|
+
const { control } = await run(
|
|
103
|
+
[
|
|
104
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
105
|
+
' await ask({ harness: "claude-sonnet", claude: { advisor: false } }, "tuned");',
|
|
106
|
+
' await ask({ harness: "claude-sonnet" }, "plain");',
|
|
107
|
+
" return null;",
|
|
108
|
+
"});",
|
|
109
|
+
"",
|
|
110
|
+
].join("\n"),
|
|
111
|
+
(seen) => seen.includes("workflow.run.finished"),
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
expect(control.starts.map((start) => start.blocks?.claude)).toEqual([{ advisor: false }, undefined]);
|
|
115
|
+
expect(Object.keys(control.starts[1] ?? {})).not.toContain("blocks");
|
|
116
|
+
}, 30_000);
|
|
117
|
+
|
|
118
|
+
test("stuckAfter of an ask() call is kept in the record of the session it starts, in milliseconds", async () => {
|
|
119
|
+
const { seen } = await run(
|
|
120
|
+
[
|
|
121
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
122
|
+
' await ask({ harness: "claude-sonnet", stuckAfter: "10m" }, "watched");',
|
|
123
|
+
' await ask({ harness: "claude-sonnet" }, "unwatched");',
|
|
124
|
+
" return null;",
|
|
125
|
+
"});",
|
|
126
|
+
"",
|
|
127
|
+
].join("\n"),
|
|
128
|
+
(seen) => seen.includes("workflow.run.finished"),
|
|
129
|
+
);
|
|
130
|
+
const started = (JSON.parse(seen) as Array<{ type: string; data: { stuckAfter?: number } }>).filter((row) => row.type === "session.started");
|
|
131
|
+
expect(started.map((row) => row.data.stuckAfter)).toEqual([600_000, undefined]);
|
|
132
|
+
}, 30_000);
|
|
133
|
+
|
|
134
|
+
test("the box an ask() call asks for is the session's: what it adds is in the box, and kept in the record of the session", async () => {
|
|
135
|
+
const { control, seen } = await run(
|
|
136
|
+
[
|
|
137
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
138
|
+
' await ask({ harness: "claude-sonnet", cwd: "/work", box: { allow: [{ net: "crates.io" }] } }, "boxed");',
|
|
139
|
+
" return null;",
|
|
140
|
+
"});",
|
|
141
|
+
"",
|
|
142
|
+
].join("\n"),
|
|
143
|
+
(seen) => seen.includes("workflow.run.finished"),
|
|
144
|
+
);
|
|
145
|
+
expect(control.starts[0]?.box?.spec.net?.map(String)).toEqual(["crates.io"]);
|
|
146
|
+
const started = (JSON.parse(seen) as Array<{ type: string; data: { box?: unknown } }>).filter((row) => row.type === "session.started");
|
|
147
|
+
expect(started.map((row) => row.data.box)).toEqual([{ allow: [{ net: "crates.io" }] }]);
|
|
148
|
+
}, 30_000);
|
|
149
|
+
|
|
150
|
+
test("an ask() call's box is held to the run's uses: wider than the default takes the run's box rule", async () => {
|
|
151
|
+
const script = (uses: string) =>
|
|
152
|
+
[
|
|
153
|
+
`export default workflow({ uses: [${uses}], output: z.json() }, async () => {`,
|
|
154
|
+
" try {",
|
|
155
|
+
' await ask({ harness: "claude-sonnet", cwd: "/work", box: { allow: [{ net: "crates.io" }] } }, "boxed");',
|
|
156
|
+
' return "started";',
|
|
157
|
+
" } catch (error) {",
|
|
158
|
+
" return String(error);",
|
|
159
|
+
" }",
|
|
160
|
+
"});",
|
|
161
|
+
"",
|
|
162
|
+
].join("\n");
|
|
163
|
+
const narrow = await run(script('"session.start.claude-sonnet"'), (seen) => seen.includes("workflow.run.finished"));
|
|
164
|
+
expect(narrow.control.starts).toHaveLength(0);
|
|
165
|
+
expect(narrow.seen).toContain("box.net@crates.io");
|
|
166
|
+
// A rule a script names is an action the host serves: here a stand-in for the box plugin's.
|
|
167
|
+
const rules = plugin("box", (api) => {
|
|
168
|
+
api.action(action("box.net", { description: "A rule.", input: z.object({ host: z.string() }), output: z.object({ held: z.literal(true) }), effect: "read", recovery: "repeat", target: (input) => input.host, run: async () => ({ held: true as const }) }));
|
|
169
|
+
});
|
|
170
|
+
const wide = await run(script('"session.start.claude-sonnet", "box.net@crates.io"'), (seen) => seen.includes("workflow.run.finished"), { options: CLAUDE }, [rules]);
|
|
171
|
+
expect(wide.control.starts[0]?.box?.spec.net?.map(String)).toEqual(["crates.io"]);
|
|
172
|
+
}, 60_000);
|
|
173
|
+
|
|
174
|
+
test("the block of another kind fails the call before a session starts, and so does a field the block does not have", async () => {
|
|
175
|
+
const { control, seen } = await run(
|
|
176
|
+
[
|
|
177
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
178
|
+
' await ask({ harness: "claude-sonnet", codex: {} }, "foreign");',
|
|
179
|
+
" return null;",
|
|
180
|
+
"});",
|
|
181
|
+
"",
|
|
182
|
+
].join("\n"),
|
|
183
|
+
(seen) => seen.includes("options block only"),
|
|
184
|
+
);
|
|
185
|
+
expect(seen).toContain('the harness \\"claude-sonnet\\" takes the \\"claude\\" options block only (asked for \\"codex\\")');
|
|
186
|
+
expect(control.starts).toEqual([]);
|
|
187
|
+
|
|
188
|
+
const typo = await run(
|
|
189
|
+
[
|
|
190
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
191
|
+
' await ask({ harness: "claude-sonnet", claude: { advisor: false, advisr: true } }, "typo");',
|
|
192
|
+
" return null;",
|
|
193
|
+
"});",
|
|
194
|
+
"",
|
|
195
|
+
].join("\n"),
|
|
196
|
+
(seen) => seen.includes("advisr"),
|
|
197
|
+
);
|
|
198
|
+
expect(typo.seen).toContain("advisr");
|
|
199
|
+
expect(typo.control.starts).toEqual([]);
|
|
200
|
+
}, 60_000);
|
|
201
|
+
|
|
202
|
+
test("a harness the sessions module binds to no options block takes no block", async () => {
|
|
203
|
+
const { control, seen } = await run(
|
|
204
|
+
[
|
|
205
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
206
|
+
' await ask({ harness: "claude-sonnet", claude: { advisor: false } }, "bare");',
|
|
207
|
+
" return null;",
|
|
208
|
+
"});",
|
|
209
|
+
"",
|
|
210
|
+
].join("\n"),
|
|
211
|
+
(seen) => seen.includes("takes no options"),
|
|
212
|
+
{},
|
|
213
|
+
);
|
|
214
|
+
expect(seen).toContain('the harness \\"claude-sonnet\\" takes no options (asked for \\"claude\\")');
|
|
215
|
+
expect(control.starts).toEqual([]);
|
|
216
|
+
}, 30_000);
|
|
@@ -0,0 +1,386 @@
|
|
|
1
|
+
// `z` as a global of a script and `ask({ schema }, prompt)`.
|
|
2
|
+
//
|
|
3
|
+
// A script describes the answer it wants with zod; the call gives the answer parsed and typed.
|
|
4
|
+
// The checks run against a fake control side that records what it was asked, and:
|
|
5
|
+
//
|
|
6
|
+
// 1. the type checker accepts a typed script and refuses a misuse of the typed answer;
|
|
7
|
+
// 2. the schema reaches the session as JSON Schema, and the call gives the parsed value;
|
|
8
|
+
// 3. an answer that does not pass the schema is never returned: the step fails and the run pauses;
|
|
9
|
+
// 4. a replay gives the recorded value without asking again; an agent call is its prompt and harness, so another schema
|
|
10
|
+
// takes the recorded answer, and another prompt asks again.
|
|
11
|
+
|
|
12
|
+
import { afterEach, expect, test } from "bun:test";
|
|
13
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
import { z } from "zod";
|
|
16
|
+
|
|
17
|
+
import { actionName, seq } from "@niwa-labs/core";
|
|
18
|
+
import type { StoredRecord } from "@niwa-labs/core";
|
|
19
|
+
import { harnessId, machineId } from "@niwa-labs/harness";
|
|
20
|
+
import type { Harness, SessionControl, SessionId } from "@niwa-labs/harness";
|
|
21
|
+
import { createNiwa, sessions, eventually, tempHome } from "@niwa-labs/testing";
|
|
22
|
+
import type { Host } from "@niwa-labs/testing";
|
|
23
|
+
import { RunFinished, workflow, workflowCheck } from "../src/index.ts";
|
|
24
|
+
import type { RunStatus } from "../src/status.ts";
|
|
25
|
+
|
|
26
|
+
const HARNESS = harnessId("fake-harness");
|
|
27
|
+
const START = actionName("workflow.start");
|
|
28
|
+
const WAIT = actionName("workflow.wait");
|
|
29
|
+
const RESUME = actionName("workflow.resume");
|
|
30
|
+
|
|
31
|
+
/** What the fake control side was asked: the question and the JSON Schema its answer must pass. */
|
|
32
|
+
interface Asked {
|
|
33
|
+
readonly prompt: string;
|
|
34
|
+
readonly schema: Record<string, unknown>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The fake control side: it records the questions and answers with what the test says. */
|
|
38
|
+
class FakeControl implements SessionControl {
|
|
39
|
+
readonly asked: Asked[] = [];
|
|
40
|
+
/** What the session answers next. */
|
|
41
|
+
reply: unknown = undefined;
|
|
42
|
+
private counter = 0;
|
|
43
|
+
readonly closed = new Set<string>();
|
|
44
|
+
keepAlive = false;
|
|
45
|
+
private readonly untilClosed = new Map<string, () => void>();
|
|
46
|
+
beforeAsk?: () => Promise<void>;
|
|
47
|
+
|
|
48
|
+
async close(id: SessionId): Promise<void> {
|
|
49
|
+
this.closed.add(id.native);
|
|
50
|
+
this.untilClosed.get(id.native)?.();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async start(): Promise<{ session: SessionId }> {
|
|
54
|
+
this.counter += 1;
|
|
55
|
+
return { session: { harness: HARNESS, native: `session-${String(this.counter)}` as never } };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async resume(): Promise<void> {
|
|
59
|
+
throw new Error("this session cannot be resumed");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async send() {
|
|
63
|
+
return { status: "started" as const, turn: 1 };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async interrupt(): Promise<void> {}
|
|
67
|
+
|
|
68
|
+
async ask<T>(
|
|
69
|
+
id: SessionId,
|
|
70
|
+
options: { prompt: { sections: Array<{ text: string }> }; schema: z.ZodType },
|
|
71
|
+
): Promise<{ session: SessionId; value: T; at: never }> {
|
|
72
|
+
await this.beforeAsk?.();
|
|
73
|
+
if (this.closed.has(id.native)) throw new Error("this conversation has closed");
|
|
74
|
+
this.asked.push({
|
|
75
|
+
prompt: options.prompt.sections.map((section) => section.text).join("\n\n"),
|
|
76
|
+
schema: z.toJSONSchema(options.schema, { io: "input" }),
|
|
77
|
+
});
|
|
78
|
+
return { session: id, value: this.reply as T, at: 0 as never };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async *life(id: SessionId): AsyncIterable<never> {
|
|
82
|
+
if (this.keepAlive && !this.closed.has(id.native)) await new Promise<void>(resolve => this.untilClosed.set(id.native, resolve));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async state(): Promise<"running" | "idle" | "unknown"> {
|
|
86
|
+
return "unknown";
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const instance: Harness = {
|
|
91
|
+
id: HARNESS,
|
|
92
|
+
kind: "fake-kind" as never,
|
|
93
|
+
machine: machineId("machine-1"),
|
|
94
|
+
home: "/tmp/fake-home",
|
|
95
|
+
sides: {
|
|
96
|
+
reading: { list: true, read: true, watch: false },
|
|
97
|
+
control: { start: true, resume: false, send: true, interrupt: true, ask: true, structuredOutput: true, launch: { model: true, effort: ["low"], modes: ["auto"] } },
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const homes: Array<{ cleanup(): Promise<void> }> = [];
|
|
102
|
+
const hosts: Host[] = [];
|
|
103
|
+
|
|
104
|
+
afterEach(async () => {
|
|
105
|
+
for (const host of hosts.splice(0)) await host.close().catch(() => {});
|
|
106
|
+
for (const home of homes.splice(0)) await home.cleanup();
|
|
107
|
+
}, 30_000);
|
|
108
|
+
|
|
109
|
+
async function served(): Promise<{ host: Host; home: string; control: FakeControl }> {
|
|
110
|
+
const home = await tempHome();
|
|
111
|
+
homes.push(home);
|
|
112
|
+
const control = new FakeControl();
|
|
113
|
+
const host = createNiwa({ home: home.path });
|
|
114
|
+
host.use(sessions({ harnesses: [{ instance, control }] }));
|
|
115
|
+
host.use(workflow({ harnesses: [HARNESS] }));
|
|
116
|
+
await host.serve();
|
|
117
|
+
hosts.push(host);
|
|
118
|
+
return { host, home: home.path, control };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function scriptIn(home: string, name: string, lines: readonly string[]): Promise<string> {
|
|
122
|
+
await mkdir(home, { recursive: true });
|
|
123
|
+
const file = join(home, name);
|
|
124
|
+
await writeFile(file, lines.join("\n"));
|
|
125
|
+
return file;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const statusOf = async (host: Host, run: string): Promise<RunStatus> =>
|
|
129
|
+
(await host.call(WAIT, { run, timeoutMs: 20_000 })) as RunStatus;
|
|
130
|
+
|
|
131
|
+
const REVIEW = [
|
|
132
|
+
'const Review = z.object({ verdict: z.enum(["approve", "changes"]), notes: z.array(z.string()), tag: z.string().default("none") });',
|
|
133
|
+
"type Review = z.infer<typeof Review>;",
|
|
134
|
+
];
|
|
135
|
+
|
|
136
|
+
test("the type checker takes a typed script, and refuses a field the schema does not have", async () => {
|
|
137
|
+
const { host, home } = await served();
|
|
138
|
+
const typed = await scriptIn(home, "typed.ts", [
|
|
139
|
+
...REVIEW,
|
|
140
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
141
|
+
' const review: Review = await ask({ harness: "fake-harness", schema: Review }, "Review it");',
|
|
142
|
+
" const first: string | undefined = review.notes[0];",
|
|
143
|
+
" const answer: { answer: string } = await ask({ harness: \"fake-harness\" }, \"free text\");",
|
|
144
|
+
" return { verdict: review.verdict, tag: review.tag, first, text: answer.answer };",
|
|
145
|
+
"});",
|
|
146
|
+
]);
|
|
147
|
+
expect(await host.call(workflowCheck, { file: typed })).toMatchObject({ ok: true, errors: [] });
|
|
148
|
+
|
|
149
|
+
const wrong = await scriptIn(home, "wrong.ts", [
|
|
150
|
+
...REVIEW,
|
|
151
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
152
|
+
' const review = await ask({ harness: "fake-harness", schema: Review }, "Review it");',
|
|
153
|
+
" return review.missing;",
|
|
154
|
+
"});",
|
|
155
|
+
]);
|
|
156
|
+
const report = await host.call(workflowCheck, { file: wrong });
|
|
157
|
+
expect(report.ok).toBe(false);
|
|
158
|
+
expect(report.errors[0]).toMatchObject({ code: "TS2339", line: 5 });
|
|
159
|
+
expect(report.errors[0]?.message).toContain("missing");
|
|
160
|
+
|
|
161
|
+
// The text answer is `{ answer }` and nothing else.
|
|
162
|
+
const text = await scriptIn(home, "text.ts", [
|
|
163
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
164
|
+
' const answer = await ask({ harness: "fake-harness" }, "free text");',
|
|
165
|
+
" return answer.verdict;",
|
|
166
|
+
"});",
|
|
167
|
+
]);
|
|
168
|
+
expect((await host.call(workflowCheck, { file: text })).errors[0]).toMatchObject({ code: "TS2339" });
|
|
169
|
+
}, 30_000);
|
|
170
|
+
|
|
171
|
+
test("the schema reaches the session as JSON Schema, and the call gives the parsed typed value", async () => {
|
|
172
|
+
const { host, home, control } = await served();
|
|
173
|
+
control.reply = { verdict: "approve", notes: ["a", "b"] };
|
|
174
|
+
const file = await scriptIn(home, "review.js", [
|
|
175
|
+
...REVIEW.slice(0, 1),
|
|
176
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
177
|
+
' const review = await ask({ harness: "fake-harness", schema: Review, label: "review" }, "Review the diff");',
|
|
178
|
+
" return review;",
|
|
179
|
+
"});",
|
|
180
|
+
]);
|
|
181
|
+
const started = (await host.call(START, { file, wait: true, timeoutMs: 20_000 })) as RunStatus;
|
|
182
|
+
// The default of the schema is applied in the script: the answer had no `tag`.
|
|
183
|
+
expect(started).toMatchObject({ state: "finished", outcome: { result: { verdict: "approve", notes: ["a", "b"], tag: "none" } } });
|
|
184
|
+
|
|
185
|
+
expect(control.asked).toHaveLength(1);
|
|
186
|
+
expect(control.asked[0]?.prompt).toContain("Review the diff");
|
|
187
|
+
expect(control.asked[0]?.schema).toMatchObject({
|
|
188
|
+
type: "object",
|
|
189
|
+
properties: { verdict: { type: "string", enum: ["approve", "changes"] }, notes: { type: "array", items: { type: "string" } } },
|
|
190
|
+
required: ["verdict", "notes"],
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
// The schema is recorded with the step, as part of its input.
|
|
194
|
+
const records = await host.records.read();
|
|
195
|
+
const intent = records.find((row: StoredRecord) => {
|
|
196
|
+
const data = row.data as { action?: string; key?: string };
|
|
197
|
+
return data.action === "session.start.fake-harness" && data.key === "label:review";
|
|
198
|
+
});
|
|
199
|
+
const Review = z.object({ verdict: z.enum(["approve", "changes"]), notes: z.array(z.string()), tag: z.string().default("none") });
|
|
200
|
+
expect((intent?.data as { input: { schema: unknown } }).input.schema).toEqual(z.toJSONSchema(Review, { io: "input" }));
|
|
201
|
+
}, 30_000);
|
|
202
|
+
|
|
203
|
+
test("an answer that does not pass the schema is not returned: the step fails and the run pauses", async () => {
|
|
204
|
+
const { host, home, control } = await served();
|
|
205
|
+
control.reply = { verdict: "maybe", notes: [] };
|
|
206
|
+
const file = await scriptIn(home, "invalid.js", [
|
|
207
|
+
...REVIEW.slice(0, 1),
|
|
208
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
209
|
+
' const review = await ask({ harness: "fake-harness", schema: Review }, "Review the diff");',
|
|
210
|
+
' return { reached: true, review };',
|
|
211
|
+
"});",
|
|
212
|
+
]);
|
|
213
|
+
const runInvalid = ((await host.call(START, { file })) as { run: string }).run;
|
|
214
|
+
const paused = await statusOf(host, runInvalid);
|
|
215
|
+
expect(paused.state).toBe("paused");
|
|
216
|
+
// The refusal is what the failed step says, in the journal.
|
|
217
|
+
const records = await host.records.read();
|
|
218
|
+
expect(JSON.stringify(records.map((row) => row.data))).toContain("does not pass its schema");
|
|
219
|
+
expect(control.asked).toHaveLength(1);
|
|
220
|
+
}, 30_000);
|
|
221
|
+
|
|
222
|
+
test("a schema that JSON Schema cannot say fails in the script, before anything is asked", async () => {
|
|
223
|
+
const { host, home, control } = await served();
|
|
224
|
+
const file = await scriptIn(home, "unsayable.js", [
|
|
225
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
226
|
+
' const when = await ask({ harness: "fake-harness", schema: z.object({ at: z.date() }) }, "When?");',
|
|
227
|
+
" return when;",
|
|
228
|
+
"});",
|
|
229
|
+
]);
|
|
230
|
+
const runUnsayable = ((await host.call(START, { file })) as { run: string }).run;
|
|
231
|
+
const status = await statusOf(host, runUnsayable);
|
|
232
|
+
expect(status.state).toBe("paused");
|
|
233
|
+
expect(JSON.stringify((await host.records.read()).map((row) => row.data))).toContain("Date cannot be represented");
|
|
234
|
+
expect(control.asked).toHaveLength(0);
|
|
235
|
+
}, 30_000);
|
|
236
|
+
|
|
237
|
+
test("a replay gives the recorded value without asking again: another schema takes it, another prompt asks again", async () => {
|
|
238
|
+
const { host, home, control } = await served();
|
|
239
|
+
control.reply = { verdict: "changes", notes: ["x"] };
|
|
240
|
+
const lines = (schema: string) => [
|
|
241
|
+
`const Review = ${schema};`,
|
|
242
|
+
"export default workflow({ output: z.json() }, async () => {",
|
|
243
|
+
' return await ask({ harness: "fake-harness", schema: Review, label: "review" }, "Review the diff");',
|
|
244
|
+
"});",
|
|
245
|
+
];
|
|
246
|
+
const file = await scriptIn(home, "replay.js", lines('z.object({ verdict: z.enum(["approve", "changes"]), notes: z.array(z.string()) })'));
|
|
247
|
+
const runReplay = ((await host.call(START, { file, wait: true, timeoutMs: 20_000 })) as { run: string }).run;
|
|
248
|
+
expect(control.asked).toHaveLength(1);
|
|
249
|
+
|
|
250
|
+
// The session would now answer otherwise; the replay does not ask it.
|
|
251
|
+
control.reply = { verdict: "approve", notes: [] };
|
|
252
|
+
await host.call(RESUME, { run: runReplay });
|
|
253
|
+
await eventually(async () => (await host.records.read()).filter((row) => row.type === "workflow.run.finished").length === 2, 20_000, "the replay should finish");
|
|
254
|
+
const finished = (await host.records.read()).filter((row) => row.type === "workflow.run.finished");
|
|
255
|
+
expect(finished[1]?.data).toMatchObject({ outcome: { result: { verdict: "changes", notes: ["x"] } } });
|
|
256
|
+
expect(control.asked).toHaveLength(1);
|
|
257
|
+
|
|
258
|
+
// An agent call is its prompt and its harness: with another schema it is the same call, and its recorded answer is taken.
|
|
259
|
+
await writeFile(file, lines('z.object({ verdict: z.string(), notes: z.array(z.string()), more: z.number().optional() })').join("\n"));
|
|
260
|
+
await host.call(RESUME, { run: runReplay });
|
|
261
|
+
await eventually(async () => (await host.records.read()).filter((row) => row.type === "workflow.run.finished").length === 3, 20_000, "the revision should finish");
|
|
262
|
+
const taken = (await host.records.read()).filter((row) => row.type === "workflow.run.finished");
|
|
263
|
+
expect(taken[2]?.data).toMatchObject({ outcome: { result: { verdict: "changes", notes: ["x"] } } });
|
|
264
|
+
expect(control.asked).toHaveLength(1);
|
|
265
|
+
expect((await host.records.read()).filter((row) => row.type === "workflow.step.reused").map((row) => (row.data as { key: string }).key)).toEqual([
|
|
266
|
+
"label:review",
|
|
267
|
+
"label:review",
|
|
268
|
+
]);
|
|
269
|
+
|
|
270
|
+
// Another prompt is another call: the session is asked.
|
|
271
|
+
await writeFile(file, lines('z.object({ verdict: z.string(), notes: z.array(z.string()) })').join("\n").replace("Review the diff", "Review the whole diff"));
|
|
272
|
+
await host.call(RESUME, { run: runReplay });
|
|
273
|
+
await eventually(async () => (await host.records.read()).filter((row) => row.type === "workflow.run.finished").length === 4, 20_000, "the changed prompt should run");
|
|
274
|
+
expect(control.asked).toHaveLength(2);
|
|
275
|
+
expect(control.asked[1]?.prompt).toContain("Review the whole diff");
|
|
276
|
+
}, 30_000);
|
|
277
|
+
|
|
278
|
+
test("a session handle replays each labeled turn and preserves the conversation", async () => {
|
|
279
|
+
const { host, home, control } = await served();
|
|
280
|
+
control.reply = { answer: "saved" };
|
|
281
|
+
const file = await scriptIn(home, "conversation.js", [
|
|
282
|
+
'export default workflow({ uses: ["session.start.fake-harness", "session.ask", "session.end"], output: z.json() }, async () => {',
|
|
283
|
+
' const conversation = session({ harness: "fake-harness", label: "conversation" });',
|
|
284
|
+
' const first = await conversation.ask("first", { label: "first" });',
|
|
285
|
+
' const next = await conversation.ask({ sections: [{ id: "followup", tier: "stable", text: "second" }] }, { label: "second", schema: z.object({ answer: z.string().transform(s => s.toUpperCase()) }) });',
|
|
286
|
+
' const { address } = await conversation;',
|
|
287
|
+
' await conversation.end({ label: "close" });',
|
|
288
|
+
' return { first, next, address };',
|
|
289
|
+
'});',
|
|
290
|
+
]);
|
|
291
|
+
const { run } = await host.call(START, { file, wait: true, timeoutMs: 20_000 }) as { run: string };
|
|
292
|
+
const status = await statusOf(host, run);
|
|
293
|
+
expect(status.outcome).toMatchObject({ status: "done", result: { first: { answer: "saved" }, next: { answer: "SAVED" }, address: { harness: HARNESS, id: "session-1" } } });
|
|
294
|
+
await host.call(RESUME, { run });
|
|
295
|
+
await eventually(async () => (await host.records.read()).filter(row => row.type === "workflow.run.finished").length === 2);
|
|
296
|
+
const rows = await host.records.read();
|
|
297
|
+
expect(rows.filter(row => row.type === "session.started")).toHaveLength(1);
|
|
298
|
+
expect(rows.filter(row => row.type === "session.ended")).toHaveLength(1);
|
|
299
|
+
expect(control.asked.map(turn => turn.prompt)).toEqual(["first", "second"]);
|
|
300
|
+
expect(rows.filter(row => row.type === "workflow.step.reused").map(row => (row.data as { key: string }).key)).toEqual(["label:conversation", "label:first", "label:second", "label:close"]);
|
|
301
|
+
}, 30_000);
|
|
302
|
+
|
|
303
|
+
test("a recomputed conversation gets fresh authority after restart and ignores a late finish from its first execution", async () => {
|
|
304
|
+
const { host: first, home, control } = await served();
|
|
305
|
+
control.keepAlive = true;
|
|
306
|
+
control.reply = "ok";
|
|
307
|
+
const file = await scriptIn(home, "execution.js", [
|
|
308
|
+
'export default workflow({ uses: ["session.start.fake-harness", "session.ask", "session.end"], output: z.json() }, async () => {',
|
|
309
|
+
' const conversation = session({ harness: "fake-harness", label: "open" });',
|
|
310
|
+
' return await conversation.ask("next", { label: "turn", schema: z.string() });',
|
|
311
|
+
'});',
|
|
312
|
+
]);
|
|
313
|
+
const { run } = await first.call(START, { file, wait: true }) as { run: string };
|
|
314
|
+
await eventually(async () => control.closed.has("session-1") && (await first.records.read()).some(row =>
|
|
315
|
+
row.type === "niwa.access.closed" && (row.data as { principal: { id: string } }).principal.id === run));
|
|
316
|
+
const invocationEnded = async (target: Host) => {
|
|
317
|
+
const rows = await target.records.read();
|
|
318
|
+
const actor = rows.filter(row => row.type === "workflow.run.started").at(-1)?.actor;
|
|
319
|
+
return actor?.kind === "handler" && rows.some(row => row.type === "niwa.invocation.state" &&
|
|
320
|
+
(row.data as { invocation: string; state: string }).invocation === actor.invocation && (row.data as { state: string }).state === "done");
|
|
321
|
+
};
|
|
322
|
+
await eventually(() => invocationEnded(first));
|
|
323
|
+
const finished = (await first.records.read()).find(row => row.type === RunFinished.name)!;
|
|
324
|
+
await first.close();
|
|
325
|
+
hosts.splice(hosts.indexOf(first), 1);
|
|
326
|
+
const host = createNiwa({ home });
|
|
327
|
+
host.use(sessions({ harnesses: [{ instance, control }] }), workflow({ harnesses: [HARNESS] }));
|
|
328
|
+
await host.serve();
|
|
329
|
+
hosts.push(host);
|
|
330
|
+
let release!: () => void;
|
|
331
|
+
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
332
|
+
let asking = false;
|
|
333
|
+
control.beforeAsk = async () => { asking = true; await gate; };
|
|
334
|
+
const attempts = await Promise.allSettled([host.call(RESUME, { run, from: "label:open" }), host.call(RESUME, { run, from: "label:open" })]);
|
|
335
|
+
expect(attempts.filter(result => result.status === "fulfilled")).toHaveLength(1);
|
|
336
|
+
await eventually(() => asking);
|
|
337
|
+
await host.emit(RunFinished, RunFinished.schema.parse(finished.data));
|
|
338
|
+
expect((await host.call(actionName("workflow.inspect"), { run }) as { status: RunStatus }).status.state).toBe("running");
|
|
339
|
+
release();
|
|
340
|
+
expect((await statusOf(host, run)).outcome).toEqual({ status: "done", result: "ok" });
|
|
341
|
+
await eventually(() => control.closed.has("session-2"));
|
|
342
|
+
await eventually(async () => {
|
|
343
|
+
const grants = (await host.records.read()).filter(row => row.type === "niwa.access.closed")
|
|
344
|
+
.map(row => (row.data as { principal: { kind: string; id: string } }).principal).filter(p => p.kind === "run");
|
|
345
|
+
return new Set(grants.map(p => p.id)).size === 2;
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
await eventually(() => invocationEnded(host));
|
|
349
|
+
await host.call(RESUME, { run, from: "label:turn" });
|
|
350
|
+
await eventually(async () => (await host.records.read()).some(row =>
|
|
351
|
+
JSON.stringify(row.data).includes("recompute from the step that opened the conversation")));
|
|
352
|
+
expect(control.asked).toHaveLength(2);
|
|
353
|
+
const expanded = (await readFile(file, "utf8")).replace('"session.end"', '"session.end", "niwa.records.read"');
|
|
354
|
+
await writeFile(file, expanded);
|
|
355
|
+
await expect(host.call(actionName("workflow.revise"), { run, file })).rejects.toThrow("niwa.records.read");
|
|
356
|
+
expect(control.asked).toHaveLength(2);
|
|
357
|
+
}, 30_000);
|
|
358
|
+
|
|
359
|
+
test("concurrent first turns share a session and another handle opens its own", async () => {
|
|
360
|
+
const { host, home, control } = await served();
|
|
361
|
+
control.reply = { answer: "saved" };
|
|
362
|
+
const file = await scriptIn(home, "handles.js", [
|
|
363
|
+
'export default workflow({ uses: ["session.start.fake-harness", "session.ask", "session.end"], output: z.json() }, async () => {',
|
|
364
|
+
' await session({ harness: "fake-harness" }).end();',
|
|
365
|
+
' const a = session({ harness: "fake-harness", label: "a" });',
|
|
366
|
+
' const b = session({ harness: "fake-harness", label: "b" });',
|
|
367
|
+
' const [first, second, opened, other] = await Promise.all([a.ask("one"), a.ask("two"), a, b]);',
|
|
368
|
+
' await Promise.all([opened.end(), opened.end()]);',
|
|
369
|
+
' try { await opened.ask("closed"); } catch (error) { return [opened.id, other.id, error.message]; }',
|
|
370
|
+
' throw new Error("asked a closed session");',
|
|
371
|
+
'});',
|
|
372
|
+
]);
|
|
373
|
+
const { run } = await host.call(START, { file, wait: true, timeoutMs: 20_000 }) as { run: string };
|
|
374
|
+
const outcome = (await statusOf(host, run)).outcome;
|
|
375
|
+
expect(outcome).toMatchObject({ status: "done", result: [expect.any(String), expect.any(String), "this workflow session has ended"] });
|
|
376
|
+
await host.call(RESUME, { run });
|
|
377
|
+
await eventually(async () => (await host.records.read()).filter(row => row.type === "workflow.run.finished").length === 2);
|
|
378
|
+
const rows = await host.records.read();
|
|
379
|
+
const sessions = rows.filter(row => row.type === "session.started").map(row => (row.data as { id: string }).id);
|
|
380
|
+
expect(new Set(sessions).size).toBe(2);
|
|
381
|
+
expect(sessions).toHaveLength(2);
|
|
382
|
+
const results = rows.filter(row => row.type === "workflow.run.finished").map(row => (row.data as { outcome: { result: string[] } }).outcome.result);
|
|
383
|
+
expect(results[0]?.slice(0, 2).sort()).toEqual([...sessions].sort());
|
|
384
|
+
expect(results[1]).toEqual(results[0]);
|
|
385
|
+
expect(control.asked.map(turn => turn.prompt)).toEqual(["one", "two"]);
|
|
386
|
+
}, 30_000);
|