@niwa-labs/harness 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +12 -2
- package/package.json +12 -4
- package/src/bundled.ts +61 -0
- package/src/capabilities.ts +267 -0
- package/src/conformance/control.ts +375 -0
- package/src/conformance/fixtures.ts +44 -0
- package/src/conformance/insert.ts +136 -0
- package/src/conformance/profile.ts +93 -0
- package/src/conformance/reading.ts +339 -0
- package/src/control.ts +448 -0
- package/src/declaration.ts +224 -0
- package/src/delivery.ts +75 -0
- package/src/discover.ts +291 -0
- package/src/index.ts +560 -0
- package/src/insert-report.ts +72 -0
- package/src/insert.ts +171 -0
- package/src/instance.ts +66 -0
- package/src/options.ts +83 -0
- package/src/reading.ts +415 -0
- package/src/reverse.ts +32 -0
- package/src/skills.ts +32 -0
- package/src/tools.ts +106 -0
- package/test/conformance.test.ts +512 -0
- package/test/declaration.test.ts +271 -0
- package/test/delivery.test.ts +30 -0
- package/test/life-event.test.ts +11 -0
- package/test/profile-conformance.test.ts +20 -0
- package/test/record-fields.test.ts +57 -0
- package/test/session-mode.test.ts +39 -0
- package/test/title.test.ts +38 -0
- package/test/tools-mode.test.ts +107 -0
- package/test/tools.test.ts +42 -0
- package/test/unknown-variant.test.ts +28 -0
- package/tsconfig.json +6 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The conformance suite of the control side.
|
|
3
|
+
*
|
|
4
|
+
* The control side starts processes, so the suite runs against whatever the kind gives it: a
|
|
5
|
+
* real installation or a fake process that implements the interface. It checks what niwa relies
|
|
6
|
+
* on: the harness names the session, the abilities it declares work, and the signs of life come.
|
|
7
|
+
*
|
|
8
|
+
* The signs of life are forward-only: a watcher sees what happens after it subscribed, never
|
|
9
|
+
* what happened before. So the suite subscribes before every action that starts a turn and
|
|
10
|
+
* waits for the turn that subscription sees end. The one exception is the first turn: it begins
|
|
11
|
+
* inside the start, before there is an id to watch, so the side keeps its signs for the first
|
|
12
|
+
* stream opened on the session, and the suite opens that stream right after the start. A turn that does not end before the deadline
|
|
13
|
+
* fails its check with a message that says so: the suite waits, but it never waits forever.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
|
|
18
|
+
import { requireAbility, supports, Unavailable, Unsupported } from "../capabilities.ts";
|
|
19
|
+
import type { ControlCapabilities } from "../capabilities.ts";
|
|
20
|
+
import { DeliveryReceiptSchema, LifeEventSchema, describeToolsMode, promptFromText } from "../control.ts";
|
|
21
|
+
import type { LifeEvent, SessionControl, ToolsMode } from "../control.ts";
|
|
22
|
+
import { excessOf } from "../declaration.ts";
|
|
23
|
+
import type { KindCapabilities } from "../declaration.ts";
|
|
24
|
+
import type { Harness } from "../instance.ts";
|
|
25
|
+
import type { SessionId } from "../reading.ts";
|
|
26
|
+
|
|
27
|
+
import { attempt, collector } from "./reading.ts";
|
|
28
|
+
import type { ConformanceReport } from "./reading.ts";
|
|
29
|
+
|
|
30
|
+
/** The control side under test, the installation that declares it, and what its kind declares. */
|
|
31
|
+
export interface ControlConformanceSides {
|
|
32
|
+
readonly instance: Harness;
|
|
33
|
+
readonly control: SessionControl;
|
|
34
|
+
/** What the kind declares it can serve; the installation is checked against it when given. */
|
|
35
|
+
readonly declared?: KindCapabilities;
|
|
36
|
+
/**
|
|
37
|
+
* How many messages the harness has been sent so far, when the side under test can tell: a
|
|
38
|
+
* refusal is then checked to have sent nothing.
|
|
39
|
+
*/
|
|
40
|
+
readonly dispatched?: () => Promise<number>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* How long one turn may take before the check that waits for it fails.
|
|
45
|
+
*
|
|
46
|
+
* The deadline is what keeps the suite from hanging on a side whose turn never ends.
|
|
47
|
+
*/
|
|
48
|
+
const TURN_DEADLINE_MS = 120_000;
|
|
49
|
+
|
|
50
|
+
export function controlConformance(
|
|
51
|
+
kind: string,
|
|
52
|
+
{ instance, control, declared, dispatched }: ControlConformanceSides,
|
|
53
|
+
): () => Promise<ConformanceReport> {
|
|
54
|
+
return async () => {
|
|
55
|
+
const { check, report } = collector(kind);
|
|
56
|
+
if (declared !== undefined) {
|
|
57
|
+
// The installation is bounded by its kind: an ability, a model choice, an effort level or
|
|
58
|
+
// a mode its kind does not declare is a promise nobody can keep.
|
|
59
|
+
const excess = excessOf(instance.sides, declared, {
|
|
60
|
+
reading: instance.sides.reading !== undefined,
|
|
61
|
+
control: true,
|
|
62
|
+
insert: instance.sides.insert !== undefined,
|
|
63
|
+
});
|
|
64
|
+
check("capabilities: the installation claims no more than the kind declares", excess.length === 0, excess.join(", "));
|
|
65
|
+
|
|
66
|
+
// A tool mode the kind declares it cannot hold is refused by the side itself, by name,
|
|
67
|
+
// before anything reaches the harness: a session must never start with tools on silently.
|
|
68
|
+
const refused: Array<["niwaOnly" | "allow", ToolsMode]> = [
|
|
69
|
+
["niwaOnly", "niwa-only"],
|
|
70
|
+
["allow", { allow: [] }],
|
|
71
|
+
];
|
|
72
|
+
for (const [name, mode] of refused) {
|
|
73
|
+
if (declared.control.tools[name].supported) continue;
|
|
74
|
+
const before = dispatched === undefined ? undefined : await dispatched();
|
|
75
|
+
let refusal: unknown;
|
|
76
|
+
try {
|
|
77
|
+
const started = await control.start({ prompt: promptFromText("the conformance suite"), tools: mode });
|
|
78
|
+
refusal = new Error(`the side started ${started.session.native}`);
|
|
79
|
+
} catch (error) {
|
|
80
|
+
refusal = error;
|
|
81
|
+
}
|
|
82
|
+
check(
|
|
83
|
+
`tools: ${describeToolsMode(mode)} is refused by the side when the kind cannot hold it`,
|
|
84
|
+
refusal instanceof Unsupported && refusal.side === "control" && refusal.ability === "start",
|
|
85
|
+
refusal instanceof Error ? refusal.message : String(refusal),
|
|
86
|
+
);
|
|
87
|
+
if (before !== undefined && dispatched !== undefined) {
|
|
88
|
+
const after = await dispatched();
|
|
89
|
+
check(`tools: a refused ${describeToolsMode(mode)} sends nothing to the harness`, after === before, `${after - before} messages were sent`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (!supports(instance, "control", "start")) {
|
|
94
|
+
check("the side declares it can start", false, "control.start is not declared");
|
|
95
|
+
return report();
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Every sign of life the suite has seen; the checks on them come at the end.
|
|
99
|
+
const signs: LifeEvent[] = [];
|
|
100
|
+
|
|
101
|
+
// the harness names the session; its identity is the harness id and that name.
|
|
102
|
+
//
|
|
103
|
+
// The start returns once the session is named; it does not wait for the first turn. The
|
|
104
|
+
// prompt is that turn, delivered like a send, and the start says what became of it.
|
|
105
|
+
const started = await attempt("start: the harness names the session", check, () =>
|
|
106
|
+
control.start({ prompt: promptFromText("the conformance suite") }),
|
|
107
|
+
);
|
|
108
|
+
if (started === undefined) return report();
|
|
109
|
+
const session = started.session;
|
|
110
|
+
check("start: the name is not empty", session.native.length > 0);
|
|
111
|
+
check(
|
|
112
|
+
"start: the session belongs to the installation",
|
|
113
|
+
session.harness === instance.id,
|
|
114
|
+
`the session is of "${session.harness}", the installation is "${instance.id}"`,
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
// The first turn: the start gives its receipt, and its end is observed like the end of any
|
|
118
|
+
// turn. It is waited for before anything else drives the session, so the checks below begin
|
|
119
|
+
// on a session that is not busy.
|
|
120
|
+
const first = started.first;
|
|
121
|
+
check(
|
|
122
|
+
"start: the first message has a receipt that says it started or queued a turn",
|
|
123
|
+
first !== undefined &&
|
|
124
|
+
DeliveryReceiptSchema.safeParse(first).success &&
|
|
125
|
+
(first.status === "started" || first.status === "queued"),
|
|
126
|
+
first === undefined ? "the start gave no receipt for its prompt" : `the receipt says "${first.status}"`,
|
|
127
|
+
);
|
|
128
|
+
if (first !== undefined) {
|
|
129
|
+
const wait = nextTurnEnd(control, session);
|
|
130
|
+
const seen = await attempt("start: the first turn ends, as the signs of life show", check, () => wait.ended);
|
|
131
|
+
if (seen !== undefined && first.turn !== undefined) {
|
|
132
|
+
check(
|
|
133
|
+
"start: the first turn that ends is the turn of the receipt",
|
|
134
|
+
seen.some((event) => event.kind === "turn-ended" && event.turn === first.turn),
|
|
135
|
+
`the receipt names turn ${first.turn}`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
signs.push(...wait.seen);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// What the side declares it can do, it does.
|
|
142
|
+
if (supports(instance, "control", "resume")) {
|
|
143
|
+
// Resuming without a prompt begins no turn, so there is no turn to wait for.
|
|
144
|
+
await attempt("resume: the session resumes by its id", check, () => control.resume(session));
|
|
145
|
+
const wait = nextTurnEnd(control, session);
|
|
146
|
+
const resumed = await attempt("resume: the session resumes with a prompt", check, async () => {
|
|
147
|
+
await control.resume(session, { prompt: promptFromText("the conformance suite") });
|
|
148
|
+
return true;
|
|
149
|
+
});
|
|
150
|
+
if (resumed !== true) wait.cancel();
|
|
151
|
+
else await attempt("resume: the turn it began has ended", check, () => wait.ended);
|
|
152
|
+
signs.push(...wait.seen);
|
|
153
|
+
}
|
|
154
|
+
if (supports(instance, "control", "send")) {
|
|
155
|
+
const wait = nextTurnEnd(control, session);
|
|
156
|
+
const receipt = await attempt("send: the message is accepted", check, () =>
|
|
157
|
+
control.send(session, "the conformance suite"),
|
|
158
|
+
);
|
|
159
|
+
if (receipt === undefined) {
|
|
160
|
+
wait.cancel();
|
|
161
|
+
} else {
|
|
162
|
+
check("send: the receipt says what happened", DeliveryReceiptSchema.safeParse(receipt).success);
|
|
163
|
+
check("send: a receipt that says the message was handled names no turn", receipt.status !== "handled" || receipt.turn === undefined);
|
|
164
|
+
// A message the harness handled itself began no turn: there is no end to wait for.
|
|
165
|
+
if (receipt.status === "handled") wait.cancel();
|
|
166
|
+
else await attempt("send: the turn it began has ended", check, () => wait.ended);
|
|
167
|
+
}
|
|
168
|
+
signs.push(...wait.seen);
|
|
169
|
+
}
|
|
170
|
+
// A side that answers by schema through `submit-answer` has no `ask` of its own to put a question to: its
|
|
171
|
+
// `ask` is the one that returns a value, which only a side with structured output has.
|
|
172
|
+
const asksItself = supports(instance, "control", "ask") && supports(instance, "control", "structuredOutput");
|
|
173
|
+
if (asksItself) {
|
|
174
|
+
const wait = nextTurnEnd(control, session);
|
|
175
|
+
const answer = await attempt("ask: an answer comes back", check, () =>
|
|
176
|
+
control.ask(session, { schema: z.string(), prompt: promptFromText("answer with a word") }),
|
|
177
|
+
);
|
|
178
|
+
if (answer === undefined) {
|
|
179
|
+
wait.cancel();
|
|
180
|
+
} else {
|
|
181
|
+
check("ask: the answer passed the schema", z.string().safeParse(answer.value).success);
|
|
182
|
+
await attempt("ask: the turn it began has ended", check, () => wait.ended);
|
|
183
|
+
}
|
|
184
|
+
signs.push(...wait.seen);
|
|
185
|
+
}
|
|
186
|
+
if (supports(instance, "control", "send") && asksItself) {
|
|
187
|
+
// A question to a busy session waits behind what runs, like a send: the message begins a
|
|
188
|
+
// turn, and the question comes straight after it, while that turn may still be running.
|
|
189
|
+
const wait = nextTurnEnd(control, session);
|
|
190
|
+
const sent = await attempt("ask behind a turn: the message that keeps the session busy is sent", check, () =>
|
|
191
|
+
control.send(session, "the conformance suite"),
|
|
192
|
+
);
|
|
193
|
+
if (sent === undefined) {
|
|
194
|
+
wait.cancel();
|
|
195
|
+
} else {
|
|
196
|
+
const answer = await attempt("ask behind a turn: a question to a busy session is answered", check, () =>
|
|
197
|
+
control.ask(session, { schema: z.string(), prompt: promptFromText("answer with a word") }),
|
|
198
|
+
);
|
|
199
|
+
if (answer !== undefined) {
|
|
200
|
+
check("ask behind a turn: the answer passed the schema", z.string().safeParse(answer.value).success);
|
|
201
|
+
}
|
|
202
|
+
await attempt("ask behind a turn: the turn that ran before the question has ended", check, () => wait.ended);
|
|
203
|
+
}
|
|
204
|
+
signs.push(...wait.seen);
|
|
205
|
+
}
|
|
206
|
+
if (supports(instance, "control", "interrupt")) {
|
|
207
|
+
// A turn to interrupt: the suite sends one and cuts it.
|
|
208
|
+
const wait = nextTurnEnd(control, session);
|
|
209
|
+
const sent = await attempt("interrupt: a message to interrupt is sent", check, () =>
|
|
210
|
+
control.send(session, "the conformance suite"),
|
|
211
|
+
);
|
|
212
|
+
if (sent === undefined) {
|
|
213
|
+
wait.cancel();
|
|
214
|
+
} else {
|
|
215
|
+
// The turn must have begun, so the interrupt has a running turn to cut. A turn that
|
|
216
|
+
// ends without its start having been reported fails here, with that reason.
|
|
217
|
+
await attempt("interrupt: the turn began before it was cut", check, () => wait.started);
|
|
218
|
+
await attempt("interrupt: a turn is interrupted", check, async () => {
|
|
219
|
+
try {
|
|
220
|
+
await control.interrupt(session);
|
|
221
|
+
} catch (error) {
|
|
222
|
+
// The turn ended before the interrupt reached it. The side said so by name, and
|
|
223
|
+
// that is its right, not a defect.
|
|
224
|
+
if (!(error instanceof Unavailable)) throw error;
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
await attempt("interrupt: the turn it cut has ended", check, () => wait.ended);
|
|
228
|
+
}
|
|
229
|
+
signs.push(...wait.seen);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// what it does not declare is refused by name.
|
|
233
|
+
for (const ability of ["resume", "send", "interrupt", "ask"] as const satisfies readonly (keyof ControlCapabilities)[]) {
|
|
234
|
+
if (supports(instance, "control", ability)) continue;
|
|
235
|
+
let refusal: unknown;
|
|
236
|
+
try {
|
|
237
|
+
requireAbility(instance, "control", ability);
|
|
238
|
+
} catch (error) {
|
|
239
|
+
refusal = error;
|
|
240
|
+
}
|
|
241
|
+
check(
|
|
242
|
+
`refusal: ${ability} is refused by name when it is not declared`,
|
|
243
|
+
refusal instanceof Error && refusal.name === "Unsupported",
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// the turn started and the turn ended are both reported. A side that cannot drive a
|
|
248
|
+
// turn has no turn to report, so the checks apply to the sides that can.
|
|
249
|
+
const drives = (["resume", "send", "ask"] as const).some((ability) =>
|
|
250
|
+
supports(instance, "control", ability),
|
|
251
|
+
);
|
|
252
|
+
if (drives) {
|
|
253
|
+
check("life: every sign is a known one", signs.every((event) => LifeEventSchema.safeParse(event).success));
|
|
254
|
+
check("life: the turn started is reported", signs.some((event) => event.kind === "turn-started"));
|
|
255
|
+
check("life: the turn ended is reported", signs.some((event) => event.kind === "turn-ended"));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const state = await attempt("state: the side says what the session is doing", check, () => control.state(session));
|
|
259
|
+
if (state !== undefined) {
|
|
260
|
+
check("state: it is one of running, idle, unknown", ["running", "idle", "unknown"].includes(state));
|
|
261
|
+
}
|
|
262
|
+
return report();
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** A wait for the end of the next turn of a session. */
|
|
267
|
+
interface TurnEnd {
|
|
268
|
+
/** The signs of life seen until the turn ended. */
|
|
269
|
+
readonly ended: Promise<LifeEvent[]>;
|
|
270
|
+
/** The sign of life that the turn began, when the turn began. */
|
|
271
|
+
readonly started: Promise<LifeEvent>;
|
|
272
|
+
/** The signs of life seen so far, also when the wait did not end in a turn. */
|
|
273
|
+
readonly seen: LifeEvent[];
|
|
274
|
+
/** Gives the wait up: the suite stops watching the session. */
|
|
275
|
+
cancel(): void;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Waits for the end of the next turn of a session.
|
|
280
|
+
*
|
|
281
|
+
* The subscription is made when this is called, before the action that starts the turn: the
|
|
282
|
+
* stream is forward-only, so a watcher that arrives after the turn ended sees nothing of it.
|
|
283
|
+
* When the deadline passes, the wait is refused with a message that says what did not happen,
|
|
284
|
+
* and the stream is closed: the suite fails the check instead of hanging.
|
|
285
|
+
*/
|
|
286
|
+
function nextTurnEnd(control: SessionControl, session: SessionId, deadlineMs = TURN_DEADLINE_MS): TurnEnd {
|
|
287
|
+
const seen: LifeEvent[] = [];
|
|
288
|
+
let settle: ((value: LifeEvent[]) => void) | undefined;
|
|
289
|
+
let refuse: ((reason: Error) => void) | undefined;
|
|
290
|
+
let giveUp: (() => void) | undefined;
|
|
291
|
+
let began: ((value: LifeEvent) => void) | undefined;
|
|
292
|
+
let beganNever: ((reason: Error) => void) | undefined;
|
|
293
|
+
let hasBegun = false;
|
|
294
|
+
|
|
295
|
+
const ended = new Promise<LifeEvent[]>((resolve, reject) => {
|
|
296
|
+
settle = resolve;
|
|
297
|
+
refuse = reject;
|
|
298
|
+
});
|
|
299
|
+
// The waits may be given up before anyone awaits them; an unhandled refusal is noise.
|
|
300
|
+
void ended.catch(() => undefined);
|
|
301
|
+
|
|
302
|
+
const started = new Promise<LifeEvent>((resolve, reject) => {
|
|
303
|
+
began = resolve;
|
|
304
|
+
beganNever = reject;
|
|
305
|
+
// The start wait ends when the end wait does: both wait for the same turn.
|
|
306
|
+
void ended.catch((error: unknown) =>
|
|
307
|
+
reject(error instanceof Error ? error : new Error(String(error))),
|
|
308
|
+
);
|
|
309
|
+
});
|
|
310
|
+
void started.catch(() => undefined);
|
|
311
|
+
|
|
312
|
+
// One promise for the whole wait: a wait given up in one turn is given up in the next.
|
|
313
|
+
const givenUp = new Promise<{ kind: "given up" }>((resolve) => {
|
|
314
|
+
giveUp = () => resolve({ kind: "given up" });
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
void (async () => {
|
|
318
|
+
const deadline = Date.now() + deadlineMs;
|
|
319
|
+
const iterator = control.life(session)[Symbol.asyncIterator]();
|
|
320
|
+
try {
|
|
321
|
+
for (;;) {
|
|
322
|
+
const remaining = deadline - Date.now();
|
|
323
|
+
if (remaining <= 0) {
|
|
324
|
+
refuse?.(new Error(`the turn did not end within ${deadlineMs} ms`));
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
// The first next() registers the watcher, before the action that starts the turn.
|
|
328
|
+
const next = await Promise.race([
|
|
329
|
+
iterator.next().then((value) => ({ kind: "event" as const, value })),
|
|
330
|
+
expiry(remaining),
|
|
331
|
+
givenUp,
|
|
332
|
+
]);
|
|
333
|
+
if (next.kind === "timeout") {
|
|
334
|
+
refuse?.(new Error(`the turn did not end within ${deadlineMs} ms`));
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
if (next.kind === "given up") {
|
|
338
|
+
refuse?.(new Error("the wait for the turn was given up"));
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
if (next.value.done === true) {
|
|
342
|
+
refuse?.(new Error("the session ended before a turn did"));
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
seen.push(next.value.value);
|
|
346
|
+
if (next.value.value.kind === "turn-started") {
|
|
347
|
+
hasBegun = true;
|
|
348
|
+
began?.(next.value.value);
|
|
349
|
+
}
|
|
350
|
+
if (next.value.value.kind === "turn-ended") {
|
|
351
|
+
// A turn that ends before its start was reported leaves nothing for `started` to wait for.
|
|
352
|
+
if (!hasBegun) beganNever?.(new Error("the turn ended without its start having been reported"));
|
|
353
|
+
settle?.(seen);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
} catch (error) {
|
|
358
|
+
refuse?.(error instanceof Error ? error : new Error(String(error)));
|
|
359
|
+
} finally {
|
|
360
|
+
// The stream is closed, but not waited for: a generator that waits inside for its next
|
|
361
|
+
// event would keep the suite waiting for it, and that is the hang this suite avoids.
|
|
362
|
+
void Promise.resolve(iterator.return?.()).catch(() => undefined);
|
|
363
|
+
}
|
|
364
|
+
})();
|
|
365
|
+
|
|
366
|
+
return { ended, started, seen, cancel: () => giveUp?.() };
|
|
367
|
+
|
|
368
|
+
/** A promise that ends when the deadline is near. */
|
|
369
|
+
function expiry(ms: number): Promise<{ kind: "timeout" }> {
|
|
370
|
+
return new Promise((resolve) => {
|
|
371
|
+
const timer = setTimeout(() => resolve({ kind: "timeout" }), ms);
|
|
372
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fixtures of the reading conformance suite.
|
|
3
|
+
*
|
|
4
|
+
* The suite runs the same four small synthetic sessions against every kind. The kind writes them
|
|
5
|
+
* in its own on-disk format; niwa never writes a harness's files, because it does not know them.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { SessionId } from "../reading.ts";
|
|
9
|
+
|
|
10
|
+
/** The four scenarios. Every kind provides all four: a skipped scenario would be a free pass. */
|
|
11
|
+
export const SCENARIOS = ["running", "nested", "compaction", "broken"] as const;
|
|
12
|
+
|
|
13
|
+
export type Scenario = (typeof SCENARIOS)[number];
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* What a writer wrote. One session per scenario, in a directory the suite created.
|
|
17
|
+
*
|
|
18
|
+
* Every scenario holds at least a human message and an answer, so the common kinds can be
|
|
19
|
+
* checked in each. On top of that:
|
|
20
|
+
*
|
|
21
|
+
* - `running` — the last line is half written, and `finish` completes it;
|
|
22
|
+
* - `nested` — a session was spawned inside; its id is `child`;
|
|
23
|
+
* - `compaction` — the harness replaced the context with a summary, `summary`;
|
|
24
|
+
* - `broken` — a whole line that the harness's format does not know.
|
|
25
|
+
*/
|
|
26
|
+
export interface FixtureResult {
|
|
27
|
+
readonly session: SessionId;
|
|
28
|
+
/** `nested`: the session spawned inside `session`. */
|
|
29
|
+
readonly child?: SessionId;
|
|
30
|
+
/** `compaction`: the summary the harness wrote. */
|
|
31
|
+
readonly summary?: string;
|
|
32
|
+
/** `running`: completes the half-written last line. */
|
|
33
|
+
readonly finish?: () => Promise<void>;
|
|
34
|
+
/** Makes the session unreadable while it is still there, so a scan of it fails. */
|
|
35
|
+
readonly makeUnreadable: () => Promise<void>;
|
|
36
|
+
/** Takes the session away: its file is gone. */
|
|
37
|
+
readonly remove: () => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Writes one scenario into `dir`, in the kind's own format, where the reading side under test
|
|
42
|
+
* looks for it.
|
|
43
|
+
*/
|
|
44
|
+
export type FixtureWriter = (dir: string, scenario: Scenario) => Promise<FixtureResult>;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The conformance suite of the insert side.
|
|
3
|
+
*
|
|
4
|
+
* The insert reports what cannot be seen from outside. The suite reads the events the side
|
|
5
|
+
* yields and holds the side to what its installation declares: an event it reports is an ability
|
|
6
|
+
* it declares, and an ability it declares is one it showed.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { supports } from "../capabilities.ts";
|
|
10
|
+
import type { InsertCapabilities } from "../capabilities.ts";
|
|
11
|
+
import type { Harness } from "../instance.ts";
|
|
12
|
+
import { InsertEventSchema } from "../insert.ts";
|
|
13
|
+
import type { HarnessInsert, InsertEvent } from "../insert.ts";
|
|
14
|
+
|
|
15
|
+
import { attempt, collector } from "./reading.ts";
|
|
16
|
+
import type { ConformanceReport } from "./reading.ts";
|
|
17
|
+
|
|
18
|
+
/** The insert side under test, and the installation that declares it. */
|
|
19
|
+
export interface InsertConformanceSides {
|
|
20
|
+
readonly instance: Harness;
|
|
21
|
+
readonly insert: HarnessInsert;
|
|
22
|
+
/**
|
|
23
|
+
* How long the suite waits for events from a stream that has neither ended nor given another,
|
|
24
|
+
* in milliseconds. A live source is silent between events; the suite reads what comes in this
|
|
25
|
+
* time and says which declared events it did not reach.
|
|
26
|
+
*/
|
|
27
|
+
readonly eventWaitMs?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The ability that each kind of event shows. */
|
|
31
|
+
const EVENT_ABILITY = {
|
|
32
|
+
"nested-spawned": "nested",
|
|
33
|
+
compacted: "compaction",
|
|
34
|
+
"opened-by-hand": "manualOpen",
|
|
35
|
+
"permission-asked": "permission",
|
|
36
|
+
"session-ended": "end",
|
|
37
|
+
} as const satisfies Record<InsertEvent["kind"], keyof InsertCapabilities>;
|
|
38
|
+
|
|
39
|
+
/** Events read at most; a side that never ends its stream must not hang the suite. */
|
|
40
|
+
const EVENT_LIMIT = 32;
|
|
41
|
+
|
|
42
|
+
/** How long the suite waits for the next event of an open stream, unless told otherwise. */
|
|
43
|
+
const EVENT_WAIT_MS = 5_000;
|
|
44
|
+
|
|
45
|
+
export function insertConformance(
|
|
46
|
+
kind: string,
|
|
47
|
+
{ instance, insert, eventWaitMs = EVENT_WAIT_MS }: InsertConformanceSides,
|
|
48
|
+
): () => Promise<ConformanceReport> {
|
|
49
|
+
return async () => {
|
|
50
|
+
const { check, report } = collector(kind);
|
|
51
|
+
|
|
52
|
+
// A side that is put into the harness has no stream to read: it reports its events itself.
|
|
53
|
+
const stream = insert.events?.bind(insert);
|
|
54
|
+
check("the side either yields events or is put into the harness", stream !== undefined || insert.install !== undefined);
|
|
55
|
+
|
|
56
|
+
// The stream was open and silent when the time ran out.
|
|
57
|
+
let waitedOut = false;
|
|
58
|
+
const events =
|
|
59
|
+
stream === undefined
|
|
60
|
+
? []
|
|
61
|
+
: await attempt("events: the side yields its events", check, async () => {
|
|
62
|
+
const seen: unknown[] = [];
|
|
63
|
+
const iterator = stream()[Symbol.asyncIterator]();
|
|
64
|
+
const deadline = Date.now() + eventWaitMs;
|
|
65
|
+
try {
|
|
66
|
+
while (seen.length < EVENT_LIMIT) {
|
|
67
|
+
const remaining = deadline - Date.now();
|
|
68
|
+
const next = await waitFor(iterator.next(), remaining);
|
|
69
|
+
if (next === undefined) {
|
|
70
|
+
waitedOut = true;
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
if (next.done === true) break;
|
|
74
|
+
seen.push(next.value);
|
|
75
|
+
}
|
|
76
|
+
} finally {
|
|
77
|
+
// The subscription is closed, but not waited for: a generator that waits inside for
|
|
78
|
+
// its next event would keep the suite waiting for it.
|
|
79
|
+
void Promise.resolve(iterator.return?.()).catch(() => undefined);
|
|
80
|
+
}
|
|
81
|
+
return seen;
|
|
82
|
+
});
|
|
83
|
+
if (events === undefined) return report();
|
|
84
|
+
|
|
85
|
+
const parsed = events.map((event) => InsertEventSchema.safeParse(event));
|
|
86
|
+
check("events: every event is a known one", parsed.every((result) => result.success));
|
|
87
|
+
const known = parsed.flatMap((result) => (result.success ? [result.data] : []));
|
|
88
|
+
|
|
89
|
+
// An event is an ability declared, and an ability declared is shown by an event. A side
|
|
90
|
+
// without a stream reports its events itself, so the pair is not checked here.
|
|
91
|
+
if (stream !== undefined) {
|
|
92
|
+
for (const [eventKind, ability] of Object.entries(EVENT_ABILITY)) {
|
|
93
|
+
const seen = known.some((event) => event.kind === eventKind);
|
|
94
|
+
const declared = supports(instance, "insert", ability);
|
|
95
|
+
check(
|
|
96
|
+
`events: ${eventKind} is reported exactly when ${ability} is declared`,
|
|
97
|
+
seen === declared,
|
|
98
|
+
declared
|
|
99
|
+
? waitedOut
|
|
100
|
+
? `${ability} is declared and no ${eventKind} came in ${eventWaitMs} ms, while the stream was still open`
|
|
101
|
+
: `${ability} is declared but no ${eventKind} came`
|
|
102
|
+
: `${eventKind} came but ${ability} is not declared`,
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The answer to a permission question goes back to the side that asked.
|
|
108
|
+
const permission = supports(instance, "insert", "permission");
|
|
109
|
+
check(
|
|
110
|
+
"answers: answerPermission is implemented exactly when permission is declared",
|
|
111
|
+
permission === (insert.answerPermission !== undefined),
|
|
112
|
+
);
|
|
113
|
+
const asked = known.find((event) => event.kind === "permission-asked");
|
|
114
|
+
if (asked !== undefined && insert.answerPermission !== undefined) {
|
|
115
|
+
const answer = insert.answerPermission.bind(insert);
|
|
116
|
+
await attempt("answers: the side carries the answer back", check, () =>
|
|
117
|
+
answer({ requestId: asked.requestId, allow: true }),
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
return report();
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** What `next` gives, or `undefined` when it gives nothing within `ms`. */
|
|
125
|
+
async function waitFor<T>(next: Promise<T>, ms: number): Promise<T | undefined> {
|
|
126
|
+
if (ms <= 0) return undefined;
|
|
127
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
128
|
+
const expiry = new Promise<undefined>((resolve) => {
|
|
129
|
+
timer = setTimeout(() => resolve(undefined), ms);
|
|
130
|
+
});
|
|
131
|
+
try {
|
|
132
|
+
return await Promise.race([next, expiry]);
|
|
133
|
+
} finally {
|
|
134
|
+
clearTimeout(timer);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The profile check of a kind: a session of one installation, in a box closed to reading, with
|
|
3
|
+
* nothing but the installation's own profile.
|
|
4
|
+
*
|
|
5
|
+
* The box is nono's: everything it may read or write is named, so a path the profile leaves out is
|
|
6
|
+
* a path the harness cannot reach. The session is given one turn that asks for a word back; the
|
|
7
|
+
* turn has to end, and the word has to be in what the reading side reads from the installation's
|
|
8
|
+
* home, which the harness wrote from inside the box. A kind whose installations pass may declare
|
|
9
|
+
* its profile verified.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { mkdtempSync, realpathSync, rmSync } from "node:fs";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
|
|
16
|
+
import { admit, boxManifest, nono, requireSandbox, sandbox } from "@niwa-labs/sandbox";
|
|
17
|
+
import type { BoxManifest } from "@niwa-labs/sandbox";
|
|
18
|
+
|
|
19
|
+
import type { HarnessBinding } from "../index.ts";
|
|
20
|
+
import { promptFromText } from "../control.ts";
|
|
21
|
+
import type { LifeEvent } from "../control.ts";
|
|
22
|
+
import type { SessionRecord } from "../reading.ts";
|
|
23
|
+
|
|
24
|
+
/** What the check saw. */
|
|
25
|
+
export interface ProfileConformanceResult {
|
|
26
|
+
/** The turn ended within the time it was given. */
|
|
27
|
+
readonly ended: boolean;
|
|
28
|
+
/** The word asked for is in the session as the reading side reads it. */
|
|
29
|
+
readonly answered: boolean;
|
|
30
|
+
/** The box the session ran in. */
|
|
31
|
+
readonly box: BoxManifest;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ProfileConformanceOptions {
|
|
35
|
+
/** The working directory of the session: read, never written. */
|
|
36
|
+
readonly cwd: string;
|
|
37
|
+
/** How long the turn may take. Default: three minutes. */
|
|
38
|
+
readonly turnMs?: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const WORD = "PONG";
|
|
42
|
+
|
|
43
|
+
/** Checks that the reading side saw the complete reply, separately from the prompt and tools. */
|
|
44
|
+
export async function hasProfileReply(records: AsyncIterable<SessionRecord>): Promise<boolean> {
|
|
45
|
+
for await (const record of records) {
|
|
46
|
+
if (record.kind === "assistant" && !record.incomplete && record.text.trim() === WORD) return true;
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Runs one installation's session in a nono box with only its own profile. */
|
|
52
|
+
export async function profileConformance(binding: HarnessBinding, options: ProfileConformanceOptions): Promise<ProfileConformanceResult> {
|
|
53
|
+
const { control, reading, profile, instance } = binding;
|
|
54
|
+
if (control === undefined || reading === undefined) throw new Error(`the installation ${instance.id} needs a control and a reading side to be checked`);
|
|
55
|
+
if (profile === undefined) throw new Error(`the installation ${instance.id} has no profile`);
|
|
56
|
+
const host = sandbox();
|
|
57
|
+
host.install(instance.id, profile);
|
|
58
|
+
// A temp directory of the session's own, as every box has: the shared temp areas are closed.
|
|
59
|
+
const temp = realpathSync(mkdtempSync(join(tmpdir(), "niwa-profile-")));
|
|
60
|
+
const plan = host.plan({ cwd: options.cwd, defaults: { allow: [] }, owner: instance.id, writes: [{ path: temp, from: "temp" }], network: true });
|
|
61
|
+
const adapter = await requireSandbox(nono());
|
|
62
|
+
const admitted = admit(plan, adapter, () => undefined);
|
|
63
|
+
const env = { TMPDIR: temp };
|
|
64
|
+
const { session } = await control.start({ prompt: promptFromText(""), cwd: options.cwd, env, box: { sandbox: adapter, spec: admitted.spec } });
|
|
65
|
+
const seen: LifeEvent[] = [];
|
|
66
|
+
const turnMs = options.turnMs ?? 180_000;
|
|
67
|
+
try {
|
|
68
|
+
const watcher = (async () => {
|
|
69
|
+
for await (const event of control.life(session)) {
|
|
70
|
+
seen.push(event);
|
|
71
|
+
if (event.kind === "turn-ended") break;
|
|
72
|
+
}
|
|
73
|
+
})().catch(() => undefined);
|
|
74
|
+
await control.send(session, `Reply with the single word ${WORD} and nothing else.`);
|
|
75
|
+
const deadline = Date.now() + turnMs;
|
|
76
|
+
while (!seen.some((event) => event.kind === "turn-ended") && Date.now() < deadline) await Bun.sleep(100);
|
|
77
|
+
const ended = seen.some((event) => event.kind === "turn-ended");
|
|
78
|
+
if (ended) await watcher;
|
|
79
|
+
let answered = false;
|
|
80
|
+
for (let attempt = 0; attempt < 20 && !answered; attempt += 1) {
|
|
81
|
+
try {
|
|
82
|
+
answered = await hasProfileReply(reading.read(session));
|
|
83
|
+
} catch {
|
|
84
|
+
// The log may not be written yet.
|
|
85
|
+
}
|
|
86
|
+
if (!answered) await Bun.sleep(250);
|
|
87
|
+
}
|
|
88
|
+
return { ended, answered, box: boxManifest(plan, admitted.rules, admitted.manifest) };
|
|
89
|
+
} finally {
|
|
90
|
+
await control.dispose?.();
|
|
91
|
+
rmSync(temp, { recursive: true, force: true });
|
|
92
|
+
}
|
|
93
|
+
}
|