@niwa-labs/testing 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Niwa Labs contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,3 @@
1
- # Temporary Holding Version
1
+ # @niwa-labs/testing
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Test helpers for niwa packages: hosts, fake harnesses and workflow runs.
package/package.json CHANGED
@@ -1,6 +1,27 @@
1
1
  {
2
2
  "name": "@niwa-labs/testing",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.0.0",
4
+ "license": "MIT",
5
+ "type": "module",
6
+ "exports": {
7
+ ".": "./src/index.ts"
8
+ },
9
+ "dependencies": {
10
+ "@anthropic-ai/claude-agent-sdk": "^0.3.325",
11
+ "@niwa-labs/client": "0.0.0",
12
+ "@niwa-labs/core": "0.0.0",
13
+ "@niwa-labs/git": "0.0.0",
14
+ "@niwa-labs/harness": "0.0.0",
15
+ "@niwa-labs/history": "0.0.0",
16
+ "@niwa-labs/host": "0.0.0",
17
+ "@niwa-labs/messages": "0.0.0",
18
+ "@niwa-labs/plugin": "0.0.0",
19
+ "@niwa-labs/requests": "0.0.0",
20
+ "@niwa-labs/sandbox": "0.0.0",
21
+ "@niwa-labs/sessions": "0.0.0",
22
+ "@niwa-labs/tasks": "0.0.0",
23
+ "@niwa-labs/timer": "0.0.0",
24
+ "@niwa-labs/workflow": "0.0.0",
25
+ "zod": "^4.6.7"
26
+ }
27
+ }
package/src/cli.ts ADDED
@@ -0,0 +1,4 @@
1
+ // What the tests of the command line need besides the shared helpers: the factories of packages a
2
+ // client may not import.
3
+
4
+ export { COCKPIT_MODULE, plugin } from "@niwa-labs/plugin";
package/src/client.ts ADDED
@@ -0,0 +1,93 @@
1
+ /**
2
+ * Client helpers of the checks: they speak to a running host through the client of the core, as
3
+ * the owner or as anyone, and read what the host did.
4
+ */
5
+
6
+ import { z } from "zod";
7
+
8
+ import { actionName, seq } from "@niwa-labs/core";
9
+ import type { StoredRecord, Token } from "@niwa-labs/core";
10
+ import { connect, readOwnerToken } from "@niwa-labs/client";
11
+ import type { NiwaApi } from "@niwa-labs/client";
12
+
13
+ /** The type the core writes for every change of an invocation's state. */
14
+ export const STATE_TYPE = "niwa.invocation.state";
15
+
16
+ /** The data of a state record. */
17
+ const InvocationState = z.object({
18
+ invocation: z.string(),
19
+ state: z.string(),
20
+ reason: z.string().optional(),
21
+ choices: z.array(z.string()).optional(),
22
+ key: z.string().optional(),
23
+ });
24
+
25
+ export type InvocationStateData = z.infer<typeof InvocationState>;
26
+
27
+ /** Read the journal from the beginning through a client subscription, into `seen`, as the owner does. */
28
+ export async function watch(home: string): Promise<{ seen: StoredRecord[]; stop: () => Promise<void> }> {
29
+ const observer = await connectAsOwner(home, "watcher");
30
+ const seen: StoredRecord[] = [];
31
+ const subscription = (async () => {
32
+ for await (const record of observer.records.since()) seen.push(record);
33
+ })().catch(() => {});
34
+ return {
35
+ seen,
36
+ stop: async () => {
37
+ await observer.close();
38
+ await subscription;
39
+ },
40
+ };
41
+ }
42
+
43
+ /** The invocation the journal talks about: the first one that was admitted ("" before that). */
44
+ export function invocationOf(seen: StoredRecord[]): string {
45
+ const admitted = seen.find((record) => record.type === "niwa.invocation.admitted");
46
+ return (admitted?.data as { invocation?: string } | undefined)?.invocation ?? "";
47
+ }
48
+
49
+ /** The last state record of an invocation (the first admitted one by default). */
50
+ export function lastState(seen: StoredRecord[], invocation = invocationOf(seen)): InvocationStateData | undefined {
51
+ let last: InvocationStateData | undefined;
52
+ for (const record of seen) {
53
+ if (record.type !== STATE_TYPE) continue;
54
+ const parsed = InvocationState.safeParse(record.data);
55
+ if (parsed.success && parsed.data.invocation === invocation) last = parsed.data;
56
+ }
57
+ return last;
58
+ }
59
+
60
+ /** The token of the owner of a home the host serves: the owner's own tools read it the same way. */
61
+ export function ownerTokenOf(home: string): Token {
62
+ const token = readOwnerToken(home);
63
+ if (token === undefined) throw new Error(`the host of ${home} has not made the owner's token`);
64
+ return token;
65
+ }
66
+
67
+ /** Connect as the owner: the owner's token, whatever the client calls itself. */
68
+ export function connectAsOwner(home: string, id = "owner-test") {
69
+ return connect({ home, id, token: ownerTokenOf(home) });
70
+ }
71
+
72
+ /** Where a run got to, as `workflow.wait` answers. */
73
+ const WaitedRun = z.object({
74
+ state: z.enum(["running", "paused", "finished"]),
75
+ reason: z.string().optional(),
76
+ result: z.unknown().optional(),
77
+ });
78
+ export type WaitedRun = z.infer<typeof WaitedRun>;
79
+
80
+ /**
81
+ * Wait for a run that must finish: it returns when the run finishes and fails at once when the run
82
+ * pauses, with the reason, instead of waiting out the deadline for a decision nobody gives.
83
+ */
84
+ export async function runFinishes(api: NiwaApi, run: string, deadlineMs: number): Promise<WaitedRun> {
85
+ const deadline = Date.now() + deadlineMs;
86
+ for (;;) {
87
+ const left = deadline - Date.now();
88
+ if (left <= 0) throw new Error(`the run ${run} did not finish within ${String(deadlineMs)} ms`);
89
+ const waited = WaitedRun.parse(await api.call(actionName("workflow.wait"), { run, timeoutMs: Math.min(left, 30_000) }));
90
+ if (waited.state === "paused") throw new Error(`the run ${run} paused: ${waited.reason ?? "no reason given"}`);
91
+ if (waited.state === "finished") return waited;
92
+ }
93
+ }
@@ -0,0 +1,118 @@
1
+ /**
2
+ * A fake harness: an installation with a control side that serves no process.
3
+ *
4
+ * It names a session for every start, records what it was asked, and answers an ask with a fixed
5
+ * value, so a check can run the sessions module, the executor or an agent against a real host
6
+ * without any program behind it. A check that needs more than this (a failing start, a stream of
7
+ * signs of life) keeps its own control side.
8
+ */
9
+
10
+ import { harnessId, harnessKind, machineId, sessionId } from "@niwa-labs/harness";
11
+ import type {
12
+ AskAnswer,
13
+ AskOptions,
14
+ DeliveryReceipt,
15
+ Harness,
16
+ LifeEvent,
17
+ SessionControl,
18
+ SessionId,
19
+ SessionState,
20
+ } from "@niwa-labs/harness";
21
+ import type { HarnessBinding } from "@niwa-labs/harness";
22
+
23
+ /** The id the fake harness installation has unless a check names another. */
24
+ export const FAKE_HARNESS = "fake-harness";
25
+
26
+ /** The control side of the fake harness: it records, it serves nothing. */
27
+ export class FakeControl implements SessionControl {
28
+ /** The prompts the sessions were started with, in order, each as the text of its sections. */
29
+ readonly started: string[] = [];
30
+ /** The questions the sessions were asked, in order. */
31
+ readonly asked: string[] = [];
32
+ /** The messages sent into sessions, in order. */
33
+ readonly sent: string[] = [];
34
+ /** What a session answers to an ask. */
35
+ answer: unknown;
36
+ /** Whether an ask blocks, never answering. */
37
+ blockAsk = false;
38
+
39
+ readonly #harness: string;
40
+
41
+ constructor(harness: string, answer: unknown) {
42
+ this.#harness = harness;
43
+ this.answer = answer;
44
+ }
45
+
46
+ /** How many sessions were started. */
47
+ get starts(): number {
48
+ return this.started.length;
49
+ }
50
+
51
+ async start(options: { prompt: { sections: { text: string }[] } }): Promise<{ session: SessionId }> {
52
+ this.started.push(options.prompt.sections.map((section) => section.text).join("\n"));
53
+ return { session: sessionId(harnessId(this.#harness), `session-${String(this.started.length)}`) };
54
+ }
55
+
56
+ async resume(): Promise<void> {}
57
+
58
+ async send(_id: SessionId, message: string): Promise<DeliveryReceipt> {
59
+ this.sent.push(message);
60
+ return { status: "started", turn: this.sent.length };
61
+ }
62
+
63
+ async interrupt(): Promise<void> {}
64
+
65
+ async ask<T>(id: SessionId, options: AskOptions<T>): Promise<AskAnswer<T>> {
66
+ this.asked.push(options.prompt.sections.map((section) => section.text).join("\n"));
67
+ if (this.blockAsk) await new Promise<void>(() => {});
68
+ return { session: id, value: this.answer as T, at: 0 as never };
69
+ }
70
+
71
+ async *life(): AsyncIterable<LifeEvent> {
72
+ // The fake has no signs of life.
73
+ }
74
+
75
+ async state(): Promise<SessionState> {
76
+ return "idle";
77
+ }
78
+ }
79
+
80
+ /** What a check may set on the fake harness. */
81
+ export interface FakeHarnessOptions {
82
+ /** The id of the installation: `session.start({ harness })` names it. */
83
+ readonly id?: string;
84
+ /** What a session answers to an ask. */
85
+ readonly answer?: unknown;
86
+ }
87
+
88
+ /** A fake installation, its control side, and the binding the sessions module takes. */
89
+ export interface FakeHarness {
90
+ readonly instance: Harness;
91
+ readonly control: FakeControl;
92
+ readonly binding: HarnessBinding;
93
+ }
94
+
95
+ /** A fake harness: an installation that can start, resume, send, ask, and nothing else. */
96
+ export function fakeHarness(options: FakeHarnessOptions = {}): FakeHarness {
97
+ const id = options.id ?? FAKE_HARNESS;
98
+ const control = new FakeControl(id, options.answer ?? { result: "done" });
99
+ const instance: Harness = {
100
+ id: harnessId(id),
101
+ kind: harnessKind("fake-kind"),
102
+ machine: machineId("machine-1"),
103
+ home: "/tmp/fake-home",
104
+ sides: {
105
+ control: {
106
+ start: true,
107
+ resume: true,
108
+ send: true,
109
+ interrupt: false,
110
+ ask: true,
111
+ structuredOutput: true,
112
+ launch: { model: false, effort: [], modes: [] },
113
+ tools: { niwaOnly: false, allow: false },
114
+ },
115
+ },
116
+ };
117
+ return { instance, control, binding: { instance, control } };
118
+ }
@@ -0,0 +1,213 @@
1
+ /**
2
+ * A fake of the SDK's `query` for the control side of the claude-sdk kind: no process runs.
3
+ *
4
+ * Each query reads its user messages and answers every one with a result after a short pause;
5
+ * the first message of a query that was not resumed is preceded by the `system/init` message that
6
+ * names the session. An interrupt ends the turn in flight with an error result at once.
7
+ *
8
+ * `firstTurnMs` sets how long the first turn of a query that was not resumed takes; `Infinity`
9
+ * makes it a turn that never ends on its own.
10
+ *
11
+ * `calls` are the tool calls the SDK asks `canUseTool` about in their turn (the first by default),
12
+ * one after another, before the turn's result. A call with `cancelAfterMs` is cancelled by the SDK that long after it
13
+ * began, as the CLI does with a control request it no longer waits for; an interrupt cancels the
14
+ * call in flight and ends the turn at once. What `canUseTool` answered is kept, in `outcomes`.
15
+ */
16
+
17
+ import { randomUUID } from "node:crypto";
18
+
19
+ import type {
20
+ Options,
21
+ PermissionResult,
22
+ SDKMessage,
23
+ SDKResultMessage,
24
+ SDKSystemMessage,
25
+ SDKUserMessage,
26
+ } from "@anthropic-ai/claude-agent-sdk";
27
+
28
+ /** The shape of the SDK's `query` that the Claude Agent SDK control side takes. */
29
+ type SdkQuery = (params: {
30
+ prompt: AsyncIterable<SDKUserMessage>;
31
+ options: Options;
32
+ }) => AsyncIterable<SDKMessage> & { interrupt(): Promise<unknown>; close(): void };
33
+
34
+ /** How long a turn takes before its result comes, unless it is interrupted. */
35
+ const TURN_MS = 20;
36
+
37
+ const USAGE: SDKResultMessage["usage"] = {
38
+ cache_creation: { ephemeral_1h_input_tokens: 0, ephemeral_5m_input_tokens: 0 },
39
+ fallback_credit: null,
40
+ output_tokens_details: { thinking_tokens: 0 },
41
+ cache_creation_input_tokens: 0,
42
+ cache_read_input_tokens: 0,
43
+ inference_geo: "",
44
+ input_tokens: 0,
45
+ iterations: [],
46
+ output_tokens: 0,
47
+ server_tool_use: { web_fetch_requests: 0, web_search_requests: 0 },
48
+ service_tier: "standard",
49
+ speed: "standard",
50
+ };
51
+
52
+ function init(session: string, model: string): SDKSystemMessage {
53
+ return {
54
+ type: "system",
55
+ subtype: "init",
56
+ apiKeySource: "none",
57
+ claude_code_version: "fake",
58
+ cwd: "/",
59
+ tools: [],
60
+ mcp_servers: [],
61
+ model,
62
+ permissionMode: "default",
63
+ slash_commands: [],
64
+ output_style: "default",
65
+ skills: [],
66
+ plugins: [],
67
+ uuid: randomUUID(),
68
+ session_id: session,
69
+ };
70
+ }
71
+
72
+ function result(session: string, text: string, structured: unknown, cut: boolean): SDKResultMessage {
73
+ const common = {
74
+ type: "result" as const,
75
+ duration_ms: 0,
76
+ duration_api_ms: 0,
77
+ num_turns: 1,
78
+ stop_reason: null,
79
+ total_cost_usd: 0,
80
+ usage: USAGE,
81
+ modelUsage: {},
82
+ permission_denials: [],
83
+ uuid: randomUUID(),
84
+ session_id: session,
85
+ };
86
+ if (cut) {
87
+ return { ...common, subtype: "error_during_execution", is_error: true, errors: ["interrupted"] };
88
+ }
89
+ return {
90
+ ...common,
91
+ subtype: "success",
92
+ is_error: false,
93
+ result: text,
94
+ ...(structured === undefined ? {} : { structured_output: structured }),
95
+ };
96
+ }
97
+
98
+ /** A tool call the fake SDK asks `canUseTool` about. */
99
+ export interface FakeCall {
100
+ readonly tool: string;
101
+ readonly input: Record<string, unknown>;
102
+ readonly title?: string;
103
+ readonly description?: string;
104
+ readonly blockedPath?: string;
105
+ /** The turn of the session that asks it, counted from 1. Default: the first. */
106
+ readonly turn?: number;
107
+ /** The SDK cancels the call this long after it began. */
108
+ readonly cancelAfterMs?: number;
109
+ }
110
+
111
+ /** The fake `query`; every query it served is counted by `served` and closed by `close`. */
112
+ export function fakeSdkQuery(
113
+ options: {
114
+ firstTurnMs?: number;
115
+ calls?: readonly FakeCall[];
116
+ /** The names the program gives the models it is asked for by an alias: `system/init` says the name, not the alias. */
117
+ models?: Readonly<Record<string, string>>;
118
+ } = {},
119
+ ): {
120
+ query: SdkQuery;
121
+ served: () => number;
122
+ open: () => number;
123
+ /** What `canUseTool` answered for each call, in order; `undefined` where the query had no `canUseTool`. */
124
+ outcomes: () => ReadonlyArray<PermissionResult | undefined>;
125
+ /** The options of every query served, in order. */
126
+ asked: () => ReadonlyArray<Parameters<SdkQuery>[0]["options"]>;
127
+ } {
128
+ const asked: Array<Parameters<SdkQuery>[0]["options"]> = [];
129
+ const outcomes: Array<PermissionResult | undefined> = [];
130
+ let served = 0;
131
+ let open = 0;
132
+ const query: SdkQuery = ({ prompt, options: sdk }) => {
133
+ served += 1;
134
+ open += 1;
135
+ asked.push(sdk);
136
+ const resumed = sdk.resume;
137
+ const session = resumed ?? randomUUID();
138
+ let cut: (() => void) | undefined;
139
+ let closed = false;
140
+
141
+ async function* messages(): AsyncGenerator<SDKMessage> {
142
+ try {
143
+ let named = resumed !== undefined;
144
+ let turn = 0;
145
+ for await (const _message of prompt) {
146
+ if (closed) return;
147
+ let turnMs = TURN_MS;
148
+ turn += 1;
149
+ if (!named) {
150
+ named = true;
151
+ turnMs = options.firstTurnMs ?? TURN_MS;
152
+ yield init(session, sdk.model === undefined ? "fake" : (options.models?.[sdk.model] ?? sdk.model));
153
+ }
154
+ let cutInCalls = false;
155
+ for (const call of (options.calls ?? []).filter((entry) => (entry.turn ?? 1) === turn)) {
156
+ if (cutInCalls) break;
157
+ const ended = new AbortController();
158
+ cut = () => {
159
+ cutInCalls = true;
160
+ ended.abort();
161
+ };
162
+ const timer = call.cancelAfterMs === undefined ? undefined : setTimeout(() => ended.abort(), call.cancelAfterMs);
163
+ outcomes.push(
164
+ await sdk.canUseTool?.(call.tool, call.input, {
165
+ signal: ended.signal,
166
+ toolUseID: `tu-${outcomes.length + 1}`,
167
+ requestId: `req-${outcomes.length + 1}`,
168
+ ...(call.title === undefined ? {} : { title: call.title }),
169
+ ...(call.description === undefined ? {} : { description: call.description }),
170
+ ...(call.blockedPath === undefined ? {} : { blockedPath: call.blockedPath }),
171
+ }) ?? undefined,
172
+ );
173
+ clearTimeout(timer);
174
+ cut = undefined;
175
+ }
176
+ const interrupted = cutInCalls || (await new Promise<boolean>((resolve) => {
177
+ const timer = Number.isFinite(turnMs) ? setTimeout(() => resolve(false), turnMs) : undefined;
178
+ cut = () => {
179
+ clearTimeout(timer);
180
+ resolve(true);
181
+ };
182
+ }));
183
+ cut = undefined;
184
+ // A question with a schema is answered with a value that passes it.
185
+ yield result(session, "ok", sdk.outputFormat === undefined ? undefined : "ok", interrupted);
186
+ }
187
+ } finally {
188
+ open -= 1;
189
+ }
190
+ }
191
+
192
+ const stream = messages();
193
+ return {
194
+ [Symbol.asyncIterator]: () => stream,
195
+ interrupt: async () => {
196
+ cut?.();
197
+ },
198
+ close: () => {
199
+ closed = true;
200
+ cut?.();
201
+ },
202
+ };
203
+ };
204
+ return { query, served: () => served, open: () => open, outcomes: () => outcomes, asked: () => asked };
205
+ }
206
+
207
+ /** The line every live Claude prompt carries: Claude Code may call its advisor, which is slow and not what a test measures. */
208
+ export const NO_ADVISOR = "Do not call the advisor tool.";
209
+
210
+ /** A prompt for a live Claude session: the text, then the line that keeps the advisor out. */
211
+ export function claudePrompt(text: string): string {
212
+ return `${text}\n\n${NO_ADVISOR}`;
213
+ }