@niwa-labs/testing 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,194 @@
1
+ /**
2
+ * A stand-in for `zcode app-server --stdio`. It logs what it is given: argv and the environment
3
+ * (`argv.log`, `env.log`), every request (`requests.log`) and every answer to a request of its
4
+ * own (`answers.log`). A turn takes 200 ms and answers "fake reply"; the content "hang" never ends
5
+ * until `session/stop`, "fail" fails, "die" kills the process, "give the word" answers with JSON,
6
+ * and "silent" ends with no answer in the event, so it is read back from the transcript. The files
7
+ * "old-subscribe", "no-subscribe", "resume-fail", "no-v4", "bad-capabilities" and "no-message-id" change what it
8
+ * accepts. A `session/send` while a turn runs is refused, as the real server refuses it; the running turn is steered
9
+ * through `v4/command` (a text that begins with "reject" is rejected). The content "refuse" is refused after a pause, "mute" is never answered (its turn runs),
10
+ * "bad ack" is answered with something that is no acceptance, "bare ack" with neither a result nor an error, and "empty" ends with no answer anywhere. Each turn is
11
+ * written to the transcript, `session/messages`, as the user message that began it and the answer it gave.
12
+ *
13
+ * It speaks the wire as the real server does, in the shapes of the snapshot niwa types it with: every event
14
+ * has its envelope and the payload its type needs, so a test that sends less says only what it means to
15
+ * (`PAYLOADS` fills the rest). At the start of a turn it also sends what the real server sends there that is
16
+ * not a turn: a request the snapshot does not know (`session/readReplyLanguage`) and a message of the v4 side
17
+ * channel. `asked.log` holds every request of its own, so a test can check each answer against the method.
18
+ *
19
+ * The content "script" plays the steps in `script.json` in the turn, as the real server would, and
20
+ * then completes the turn: `{ask: {method, params}, again?: n}` is a request of its own to the client
21
+ * (announced again under a new id `n` times, as the real server does while it waits) which the turn
22
+ * waits to be answered, unless `wait` is false; `{event: {type, payload}}` is a session event; `{ms}` is a pause. A
23
+ * `session/stop` resolves what the turn still waits for, with the `permission.resolved` event the
24
+ * real server sends, and ends the turn as cancelled.
25
+ */
26
+ export const ZCODE_FAKE = `#!/usr/bin/env bun
27
+ const { appendFileSync, existsSync } = await import("node:fs");
28
+ const dir = import.meta.dir;
29
+ const log = (name, value) => appendFileSync(dir + "/" + name, JSON.stringify(value) + "\\n");
30
+ log("argv.log", process.argv.slice(2));
31
+ log("env.log", { id: process.env.NIWA_ID, pid: process.pid, storage: process.env.ZCODE_STORAGE_DIR, db: process.env.ZCODE_SESSION_DB_PATH });
32
+ const out = (message) => process.stdout.write(JSON.stringify(message) + "\\n");
33
+ const subscribeKind = () => (existsSync(dir + "/old-subscribe") ? "desktop" : "desktop-continuous");
34
+
35
+ let session = "sess_fake-1";
36
+ let active;
37
+ let turnNumber = 0;
38
+ const transcript = [];
39
+ let requestsOfServer = 0;
40
+ const askClient = (method, params) => {
41
+ requestsOfServer += 1;
42
+ log("asked.log", { id: "server-" + requestsOfServer, method });
43
+ out({ id: "server-" + requestsOfServer, method, params });
44
+ return "server-" + requestsOfServer;
45
+ };
46
+ const answerWaiters = new Map();
47
+ const unanswered = new Map();
48
+ const answered = (id) => new Promise((resolve) => { answerWaiters.set(id, resolve); });
49
+ async function playScript(userId) {
50
+ const steps = JSON.parse(await Bun.file(dir + "/script.json").text());
51
+ for (const step of steps) {
52
+ if (active === undefined) return;
53
+ if (step.ms !== undefined) await Bun.sleep(step.ms);
54
+ else if (step.event !== undefined) event(step.event.type, step.event.payload);
55
+ else if (step.ask !== undefined) {
56
+ const first = askClient(step.ask.method, step.ask.params);
57
+ unanswered.set(first, step.ask.params.requestId);
58
+ const done = answered(first).then(() => unanswered.delete(first));
59
+ for (let again = 0; again < (step.again ?? 0); again += 1) askClient(step.ask.method, step.ask.params);
60
+ if (step.wait !== false) await done;
61
+ }
62
+ }
63
+ if (active !== undefined) {
64
+ active = undefined;
65
+ transcript.push(message("assistant", "fake reply", "msg-assistant-" + turnNumber, userId));
66
+ event("turn.completed", { response: "fake reply", resultType: "success" });
67
+ }
68
+ }
69
+ let eventSeq = 0;
70
+ /** What the snapshot needs in the payload of an event, so that a test names only what it cares about. */
71
+ const PAYLOADS = {
72
+ "turn.started": { turnNumber: 0, input: "" },
73
+ "turn.steerQueued": { pendingInputId: "pending-1", input: "more", inputPreview: "more", inputSize: 4, targetTurnId: "turn-1", queueLength: 1 },
74
+ "model.streaming": { kind: "text_delta" },
75
+ "turn.completed": { response: "", tokenCount: 0, toolCallCount: 0, duration: 0, resultType: "success" },
76
+ "turn.failed": { turnPhase: "execution" },
77
+ };
78
+ const event = (type, payload) => {
79
+ eventSeq += 1;
80
+ out({ method: "session/event", params: { eventId: "event-" + eventSeq, sessionId: session, seq: eventSeq, timestamp: Date.now(), deliveryKind: "desktop-continuous", type, payload: { ...PAYLOADS[type], ...payload } } });
81
+ };
82
+ const snapshot = () => ({ protocol: { name: "ZCode Protocol", version: 1 }, session: { sessionId: session, mode: "build", model: { providerId: "fake", modelId: "default-model" } } });
83
+ const workspace = { workspacePath: dir, workspaceKey: dir };
84
+
85
+ const permissionParams = (requestId, toolName, input) => ({
86
+ requestId, sessionId: session, toolCallId: "call-" + requestId, toolName, reason: "the tool wants to run", riskLevel: "medium", input,
87
+ options: [
88
+ { optionId: "allow_once", kind: "allow_once", name: "Allow once", response: { decision: "allow", reason: "Approved once" } },
89
+ { optionId: "allow_project", kind: "allow_always", name: "Always allow in this project", description: "Do not ask again", response: { decision: "allow", reason: "Approved for this project", permissionUpdates: [{ type: "addRules", behavior: "allow", rules: [{ toolName }] }] } },
90
+ { optionId: "deny", kind: "deny", name: "Deny", response: { decision: "deny", reason: "The user does not want this." } },
91
+ ],
92
+ });
93
+
94
+ function turn(content) {
95
+ active = content;
96
+ turnNumber += 1;
97
+ const userId = "msg-user-" + turnNumber;
98
+ transcript.push(message("user", content, userId));
99
+ askClient("session/requestRuntimePreferences", { sessionId: session, scope: "user-execution" });
100
+ askClient("interaction/requestOfficialMcpAuthHeaders", { requestId: "official-mcp-auth:1", workspace, pluginId: "zcode-site@zcode-plugins-official", mcpKey: "sites", targetOrigin: "https://zcode.z.ai" });
101
+ askClient("session/readReplyLanguage", { sessionId: session });
102
+ out({ method: "v4/conversation/contextCompositionChanged", params: { sessionId: session, revision: 1 } });
103
+ if (content.includes("needs permission")) askClient("interaction/requestPermission", permissionParams("perm-1", "Bash", { command: "ls" }));
104
+ out({ method: "state.updated", params: { type: "state.updated", scope: "session", sessionId: session, revision: 1, patch: { status: "running" } } });
105
+ event("turn.started", { input: content, ...(existsSync(dir + "/no-message-id") ? {} : { messageId: userId }) });
106
+ event("model.streaming", { delta: "x", kind: "text_delta" });
107
+ if (content === "hang") return;
108
+ if (content === "script") return void playScript(userId);
109
+ if (content === "die") process.exit(3);
110
+ const settle = (name, payload) => {
111
+ if (active === undefined) return;
112
+ active = undefined;
113
+ // What the turn wrote into the transcript: its answer, unless it left none.
114
+ const written = content.startsWith("empty") || name !== "turn.completed" ? undefined : content.startsWith("silent") ? "from the transcript" : payload.response;
115
+ if (written !== undefined && written !== "") transcript.push(message("assistant", written, "msg-assistant-" + turnNumber, userId));
116
+ event(name, payload);
117
+ };
118
+ if (content.startsWith("fail")) return void setTimeout(() => settle("turn.failed", { error: { type: "execution", message: "model exploded" } }), 50);
119
+ const response = content.includes("give the fenced word")
120
+ ? "Here it is: \`\`\`json\\n{\\"word\\":\\"fenced\\"}\\n\`\`\`"
121
+ : content.includes("give the word") ? '{"word":"ok"}'
122
+ : content.startsWith("silent") || content.startsWith("empty") ? ""
123
+ : "fake reply";
124
+ setTimeout(() => {
125
+ settle("turn.completed", { response, resultType: "success" });
126
+ if (content === "stray") setTimeout(() => event("turn.completed", { response, resultType: "success" }), 30);
127
+ }, content === "slow" || content === "mute slow" ? 600 : 200);
128
+ }
129
+
130
+ const message = (role, text, messageId, parent) => ({
131
+ info: role === "user"
132
+ ? { messageId, sessionId: session, role, time: { created: 1 }, agent: "build" }
133
+ : { messageId, sessionId: session, role, time: { created: 2 }, parentMessageId: parent, agent: "build", path: { cwd: dir, root: dir }, cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } },
134
+ parts: [{ partId: "part-" + messageId, sessionId: session, messageId, type: "text", text }],
135
+ });
136
+
137
+ function handle({ id, method, params }) {
138
+ const ok = (result) => out({ id, result });
139
+ switch (method) {
140
+ case "runtime/capabilities": return ok(existsSync(dir + "/bad-capabilities") ? "pong" : { independentPlanState: true });
141
+ case "session/create": askClient("session/requestRuntimePreferences", { sessionId: session, scope: "runtime-materialization" }); return ok(snapshot());
142
+ case "session/resume":
143
+ if (existsSync(dir + "/resume-fail")) return out({ id, error: { code: -32603, message: "no such session" } });
144
+ session = params.sessionId; return ok(snapshot());
145
+ case "session/subscribe":
146
+ if (existsSync(dir + "/no-subscribe") || params.deliveryKind !== subscribeKind()) return out({ id, error: { code: -32602, message: "invalid_value in deliveryKind" } });
147
+ return ok({ sessionId: params.sessionId, eventSeq: 0, events: [] });
148
+ case "session/send":
149
+ // The real server takes no input while a prompt runs: a steer goes through v4/command.
150
+ if (active !== undefined) return out({ id, error: { code: -32010, message: "A prompt is already running for this session" } });
151
+ if (params.content.startsWith("refuse")) return void setTimeout(() => out({ id, error: { code: -32010, message: "the server refuses this message" } }), 100);
152
+ if (params.content.startsWith("bad ack")) out({ id, result: { ok: true } });
153
+ else if (params.content.startsWith("bare ack")) out({ id });
154
+ else if (!params.content.startsWith("mute")) ok({ accepted: true, sessionId: params.sessionId, stateRevision: 1 });
155
+ return void turn(params.content);
156
+ case "v4/command": {
157
+ if (existsSync(dir + "/no-v4")) return out({ id, error: { code: -32601, message: "method not found" } });
158
+ if (params.type !== "sendText" || params.payload.text.startsWith("reject")) return out({ id, result: { commandId: params.commandId, status: "rejected", reasonCode: "proto.unsupported", revisionAtDecision: 1 } });
159
+ if (active === undefined) {
160
+ out({ id, result: { commandId: params.commandId, status: "accepted", revisionAtDecision: 1, result: { type: "inputAccepted", delivery: "startNow", inputId: params.commandId } } });
161
+ return void turn(params.payload.text);
162
+ }
163
+ out({ id, result: { commandId: params.commandId, status: "accepted", revisionAtDecision: 1, result: { type: "inputAccepted", delivery: "queue", inputId: params.commandId } } });
164
+ return void event("turn.steerQueued", { delivery: "guide" });
165
+ }
166
+ case "session/stop":
167
+ ok({});
168
+ if (active !== undefined) {
169
+ active = undefined;
170
+ for (const requestId of unanswered.values()) event("permission.resolved", { requestId, toolCallId: "call-" + requestId });
171
+ setTimeout(() => event("turn.completed", { resultType: "cancelled", response: "" }), 50);
172
+ }
173
+ return;
174
+ case "session/messages":
175
+ return ok({ messages: transcript });
176
+ default: return out({ id, error: { code: -32601, message: "method not found" } });
177
+ }
178
+ }
179
+
180
+ let buffer = "";
181
+ process.stdin.setEncoding("utf8");
182
+ process.stdin.on("data", (chunk) => {
183
+ buffer += chunk;
184
+ for (let at = buffer.indexOf("\\n"); at !== -1; at = buffer.indexOf("\\n")) {
185
+ const line = buffer.slice(0, at).trim();
186
+ buffer = buffer.slice(at + 1);
187
+ if (line === "") continue;
188
+ const frame = JSON.parse(line);
189
+ if (frame.method === undefined) { log("answers.log", frame); answerWaiters.get(frame.id)?.(); }
190
+ else { log("requests.log", frame); handle(frame); }
191
+ }
192
+ });
193
+ process.stdin.on("end", () => process.exit(0));
194
+ `;
package/src/helpers.ts ADDED
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Pure helpers of the checks: they need no host and no client. A question's option, an answer
3
+ * that picks options, the gate of the live tests, and a web surface's cookie.
4
+ */
5
+
6
+ import { OptionIdSchema } from "@niwa-labs/core";
7
+ import type { Answer, QuestionOption } from "@niwa-labs/core";
8
+
9
+ /** Tests that run a real external CLI or model are skipped unless `NIWA_LIVE=1` (`bun run test:live`). */
10
+ export const live = process.env.NIWA_LIVE === "1";
11
+
12
+ /** A plain option of a question. */
13
+ export const opt = (id: string, label = id): QuestionOption => ({ id: OptionIdSchema.parse(id), label, flavor: "plain" });
14
+
15
+ /** The answer that picks these options. */
16
+ export const pick = (...ids: string[]): Answer => ({ ids: ids.map((id) => OptionIdSchema.parse(id)) });
17
+
18
+ /** The cookies the web surfaces under test gave, by their origin. */
19
+ const webCookies = new Map<string, string>();
20
+
21
+ /** Log in to a web surface by its link, as a browser does: the cookie is kept for `webFetch`. */
22
+ export async function loginWeb(web: { readonly url: string; readonly loginUrl: string }): Promise<void> {
23
+ const response = await fetch(web.loginUrl, { redirect: "manual" });
24
+ const cookie = response.headers.get("set-cookie")?.split(";")[0];
25
+ if (response.status !== 302 || cookie === undefined) {
26
+ throw new Error(`the web surface did not log in: ${String(response.status)}`);
27
+ }
28
+ webCookies.set(new URL(web.url).origin, cookie);
29
+ }
30
+
31
+ /** `fetch` with the cookie of a web surface that `loginWeb` logged in to. */
32
+ export function webFetch(url: string, init: RequestInit = {}): Promise<Response> {
33
+ const cookie = webCookies.get(new URL(url).origin);
34
+ if (cookie === undefined) return fetch(url, init);
35
+ const headers = new Headers(init.headers);
36
+ headers.set("cookie", cookie);
37
+ return fetch(url, { ...init, headers });
38
+ }