@niwa-labs/harness 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,339 @@
1
+ /**
2
+ * The conformance suite of the reading side.
3
+ *
4
+ * A third party writes a reading side for a new harness, runs this suite, and it counts as
5
+ * compatible. The suite knows no harness format: the kind writes four small synthetic sessions
6
+ * (fixtures.ts) and the suite checks what the side makes of them.
7
+ */
8
+
9
+ import { mkdtemp, rm } from "node:fs/promises";
10
+ import { tmpdir } from "node:os";
11
+ import { join } from "node:path";
12
+
13
+ import { requireAbility, supports } from "../capabilities.ts";
14
+ import type { ReadingCapabilities } from "../capabilities.ts";
15
+ import type { Harness } from "../instance.ts";
16
+ import type { Coverage, Locator, SessionId, SessionReading, SessionRecord } from "../reading.ts";
17
+ import { sameSession } from "../reading.ts";
18
+
19
+ import { SCENARIOS } from "./fixtures.ts";
20
+ import type { FixtureResult, FixtureWriter, Scenario } from "./fixtures.ts";
21
+
22
+ /** The outcome of one check. */
23
+ export interface ConformanceResult {
24
+ readonly name: string;
25
+ readonly passed: boolean;
26
+ /** What was found, when it failed. */
27
+ readonly detail?: string;
28
+ }
29
+
30
+ /** The report of a suite, the same for every side. */
31
+ export interface ConformanceReport {
32
+ /** The name of the kind that was checked. */
33
+ readonly kind: string;
34
+ readonly results: readonly ConformanceResult[];
35
+ readonly passed: boolean;
36
+ }
37
+
38
+ /** Records a check. Shared by the suites of all three sides. */
39
+ export type Check = (name: string, passed: boolean, detail?: string) => void;
40
+
41
+ /** A collector of checks and the report it makes. */
42
+ export function collector(kind: string): {
43
+ readonly check: Check;
44
+ readonly report: () => ConformanceReport;
45
+ } {
46
+ const results: ConformanceResult[] = [];
47
+ return {
48
+ check: (name, passed, detail) => {
49
+ results.push(passed || detail === undefined ? { name, passed } : { name, passed, detail });
50
+ },
51
+ report: () => ({ kind, results, passed: results.every((result) => result.passed) }),
52
+ };
53
+ }
54
+
55
+ /** Runs a call of the side; a throw is a failed check, not a crash of the suite. */
56
+ export async function attempt<T>(name: string, check: Check, call: () => Promise<T>): Promise<T | undefined> {
57
+ try {
58
+ const value = await call();
59
+ check(name, true);
60
+ return value;
61
+ } catch (error) {
62
+ check(name, false, describe(error));
63
+ return undefined;
64
+ }
65
+ }
66
+
67
+ /** The reading side under test, and the installation that declares it. */
68
+ export interface ConformanceSides {
69
+ readonly instance: Harness;
70
+ readonly reading: SessionReading;
71
+ }
72
+
73
+ /**
74
+ * The suite of a reading side.
75
+ *
76
+ * The writer puts a scenario's fixtures into a directory the suite made, then `open` is given
77
+ * that directory and returns the side bound to it: the side must find the fixtures there.
78
+ */
79
+ export function readingConformance(
80
+ kind: string,
81
+ open: (dir: string) => ConformanceSides,
82
+ writer: FixtureWriter,
83
+ ): () => Promise<ConformanceReport> {
84
+ return async () => {
85
+ const { check, report } = collector(kind);
86
+ let instance: Harness | undefined;
87
+ let reading: SessionReading | undefined;
88
+
89
+ for (const scenario of SCENARIOS) {
90
+ const dir = await mkdtemp(join(tmpdir(), `niwa-conformance-${scenario}-`));
91
+ try {
92
+ const written = await writer(dir, scenario);
93
+ const sides = open(dir);
94
+ instance = sides.instance;
95
+ reading = sides.reading;
96
+ const declared = supports(instance, "reading", "list") && supports(instance, "reading", "read");
97
+ check(`${scenario}: the side declares it can list and read`, declared);
98
+ if (!declared) continue;
99
+ await checkScenario(scenario, sides, written, check);
100
+ } catch (error) {
101
+ check(`${scenario}: the suite ran to the end`, false, describe(error));
102
+ } finally {
103
+ await rm(dir, { recursive: true, force: true });
104
+ }
105
+ }
106
+
107
+ if (instance !== undefined && reading !== undefined) checkDeclarations({ instance, reading }, check);
108
+ return report();
109
+ };
110
+ }
111
+
112
+ async function checkScenario(
113
+ scenario: Scenario,
114
+ { reading }: ConformanceSides,
115
+ written: FixtureResult,
116
+ check: Check,
117
+ ): Promise<void> {
118
+ const { session } = written;
119
+ const at = (what: string): string => `${scenario}: ${what}`;
120
+
121
+ // The list finds the session.
122
+ const listed: SessionId[] = [];
123
+ for await (const info of reading.list()) listed.push(info.id);
124
+ check(
125
+ at("the list finds the session"),
126
+ listed.some((id) => sameSession(id, session)),
127
+ `the list holds ${listed.map((id) => `${id.harness}/${id.native}`).join(", ") || "nothing"}`,
128
+ );
129
+
130
+ // A session with records that nobody has read is "not scanned", never "empty".
131
+ await checkCoverageIs(at("a session never read is not-scanned"), reading, "not-scanned", check, session);
132
+
133
+ const first = await readAll(reading, session);
134
+ const finished = scenario === "running" ? await checkRunning(reading, written, first, check) : first;
135
+
136
+ // The common kinds, in the final state of the session.
137
+ const kinds = new Set<string>(finished.map((record) => record.kind));
138
+ check(
139
+ at("the common kinds come back"),
140
+ kinds.has("user") && kinds.has("assistant"),
141
+ `the kinds are ${[...kinds].join(", ") || "none"}`,
142
+ );
143
+ check(at("every record is whole"), finished.every((record) => !record.incomplete));
144
+ check(
145
+ at("a read from the start is not marked as one that started over"),
146
+ finished.every((record) => record.reanchored !== true),
147
+ );
148
+ check(at("what a user message says is inserted lies inside its text"), finished.every(insertionsHold));
149
+ check(at("an image mark lies inside the content it stood in"), finished.every(imageMarksHold));
150
+ check(
151
+ at("the paths of an edit call begin with its target"),
152
+ finished.every((record) => record.kind !== "tool-call" || record.paths === undefined || record.paths[0] === record.target),
153
+ );
154
+ check(
155
+ at("locators are distinct"),
156
+ new Set(finished.map((record) => record.locator)).size === finished.length,
157
+ );
158
+
159
+ switch (scenario) {
160
+ case "nested": {
161
+ const nested = finished.find((record) => record.kind === "nested");
162
+ check(
163
+ at("the nested session is the child"),
164
+ nested?.kind === "nested" && written.child !== undefined && sameSession(nested.child, written.child),
165
+ nested === undefined ? "no nested record came back" : "the nested record names another session",
166
+ );
167
+ break;
168
+ }
169
+ case "compaction": {
170
+ const compaction = finished.find((record) => record.kind === "compaction");
171
+ check(
172
+ at("the compaction keeps the summary"),
173
+ compaction?.kind === "compaction" && compaction.summary === written.summary,
174
+ compaction === undefined ? "no compaction record came back" : "the summary differs",
175
+ );
176
+ break;
177
+ }
178
+ case "broken":
179
+ await checkUnknown(reading, session, finished, check);
180
+ break;
181
+ case "running":
182
+ break;
183
+ }
184
+
185
+ // Reading from the locator of any record yields exactly what follows it.
186
+ let resumed = true;
187
+ let resumeDetail: string | undefined;
188
+ for (const [index, record] of finished.entries()) {
189
+ const after = (await readFrom(reading, session, record.locator)).map((r) => r.locator);
190
+ const expected = finished.slice(index + 1).map((r) => r.locator);
191
+ if (after.length !== expected.length || after.some((locator, i) => locator !== expected[i])) {
192
+ resumed = false;
193
+ resumeDetail = `from record ${index} the side gave ${after.length} records, expected ${expected.length}`;
194
+ break;
195
+ }
196
+ }
197
+ check(at("reading from a locator yields what follows"), resumed, resumeDetail);
198
+
199
+ // The four coverage states are four different answers: not-scanned above, then these.
200
+ const last = finished[finished.length - 1];
201
+ if (last !== undefined) {
202
+ await checkCoverageIs(at("a range read and empty is empty"), reading, "empty", check, session, last.locator);
203
+ }
204
+ await written.makeUnreadable();
205
+ await checkCoverageIs(at("a session that cannot be scanned is failed"), reading, "failed", check, session);
206
+ await written.remove();
207
+ await checkCoverageIs(at("a session whose file is gone is unavailable"), reading, "unavailable", check, session);
208
+ }
209
+
210
+ /**
211
+ * The half-written last line. Returns the records of the session once the line is whole.
212
+ *
213
+ * Before: whole records, then only marked ones at the end. After the writer completes the line:
214
+ * reading from the last whole record gives the finished line, and nothing before it changed.
215
+ */
216
+ async function checkRunning(
217
+ reading: SessionReading,
218
+ written: FixtureResult,
219
+ before: readonly SessionRecord[],
220
+ check: Check,
221
+ ): Promise<readonly SessionRecord[]> {
222
+ const { session } = written;
223
+ const firstMarked = before.findIndex((record) => record.incomplete);
224
+ const whole = firstMarked < 0 ? before : before.slice(0, firstMarked);
225
+ const tail = firstMarked < 0 ? [] : before.slice(firstMarked);
226
+
227
+ check("running: the half-written last line is marked incomplete", tail.length > 0);
228
+ check(
229
+ "running: nothing whole comes after the half-written line",
230
+ tail.every((record) => record.incomplete),
231
+ );
232
+ check("running: whole records come before the half-written line", whole.length > 0);
233
+
234
+ if (written.finish === undefined) {
235
+ check("running: the fixture can finish the line", false, "the writer gave no finish");
236
+ return before;
237
+ }
238
+ await written.finish();
239
+ const after = await readAll(reading, session);
240
+
241
+ check("running: once the line is whole nothing is marked", after.every((record) => !record.incomplete));
242
+ check(
243
+ "running: the records before the half-written line did not change",
244
+ whole.every((record, index) => after[index]?.locator === record.locator && after[index]?.kind === record.kind),
245
+ );
246
+ const anchor = whole[whole.length - 1];
247
+ if (anchor !== undefined) {
248
+ const resumed = await readFrom(reading, session, anchor.locator);
249
+ check(
250
+ "running: reading continues from the last whole record",
251
+ resumed.length > 0 &&
252
+ resumed.every((record) => !record.incomplete) &&
253
+ resumed.length === after.length - whole.length,
254
+ `${resumed.length} records came back, ${after.length - whole.length} expected`,
255
+ );
256
+ }
257
+ return after;
258
+ }
259
+
260
+ /** An unfamiliar whole line is "unknown" with its raw data, and reading does not stop at it. */
261
+ async function checkUnknown(
262
+ reading: SessionReading,
263
+ session: SessionId,
264
+ records: readonly SessionRecord[],
265
+ check: Check,
266
+ ): Promise<void> {
267
+ const unknown = records.find((record) => record.kind === "unknown");
268
+ check("broken: the unfamiliar line comes back as unknown", unknown !== undefined);
269
+ if (unknown?.kind !== "unknown") return;
270
+ check("broken: the unknown record keeps its raw data", unknown.raw !== undefined);
271
+ const raw = await reading.raw(session, unknown.locator);
272
+ check("broken: raw gives the harness's own record", raw !== undefined && reading.rawSchema.safeParse(raw).success);
273
+ }
274
+
275
+ /** The ranges a user message marks as inserted lie inside its text and do not overlap. */
276
+ function insertionsHold(record: SessionRecord): boolean {
277
+ if (record.kind !== "user" || record.inserted === undefined) return true;
278
+ const ranges = [...record.inserted].sort((a, b) => a.from - b.from);
279
+ return ranges.every((range, index) => range.to <= record.text.length && (index === 0 || ranges[index - 1]!.to <= range.from));
280
+ }
281
+
282
+ /** The images of a user message stand within its text, and those of a result within its output. */
283
+ function imageMarksHold(record: SessionRecord): boolean {
284
+ if (record.kind === "user") return (record.images ?? []).every((image) => image.at <= record.text.length);
285
+ if (record.kind !== "tool-result" || record.images === undefined) return true;
286
+ const size = Array.isArray(record.output) ? record.output.length : typeof record.output === "string" ? record.output.length : 0;
287
+ return record.images.every((image) => image.at <= size);
288
+ }
289
+
290
+ /** The refusals: what a side declares it implements, and what it does not it is refused. */
291
+ function checkDeclarations({ instance, reading }: ConformanceSides, check: Check): void {
292
+ const watching = supports(instance, "reading", "watch");
293
+ check(
294
+ "refusal: watch is declared exactly when it is implemented",
295
+ watching === (typeof reading.watch === "function"),
296
+ watching ? "watch is declared but not implemented" : "watch is implemented but not declared",
297
+ );
298
+ for (const ability of ["list", "read", "watch"] as const satisfies readonly (keyof ReadingCapabilities)[]) {
299
+ if (supports(instance, "reading", ability)) continue;
300
+ let refusal: unknown;
301
+ try {
302
+ requireAbility(instance, "reading", ability);
303
+ } catch (error) {
304
+ refusal = error;
305
+ }
306
+ check(
307
+ `refusal: ${ability} is refused by name when it is not declared`,
308
+ refusal instanceof Error && refusal.name === "Unsupported",
309
+ );
310
+ }
311
+ }
312
+
313
+ async function checkCoverageIs(
314
+ name: string,
315
+ reading: SessionReading,
316
+ expected: Coverage["status"],
317
+ check: Check,
318
+ session: SessionId,
319
+ from?: Locator,
320
+ ): Promise<void> {
321
+ const coverage = await reading.coverage(session, from);
322
+ check(name, coverage.status === expected, `the coverage says ${coverage.status}`);
323
+ }
324
+
325
+ async function readAll(reading: SessionReading, id: SessionId): Promise<SessionRecord[]> {
326
+ const records: SessionRecord[] = [];
327
+ for await (const record of reading.read(id)) records.push(record);
328
+ return records;
329
+ }
330
+
331
+ async function readFrom(reading: SessionReading, id: SessionId, from: Locator): Promise<SessionRecord[]> {
332
+ const records: SessionRecord[] = [];
333
+ for await (const record of reading.read(id, from)) records.push(record);
334
+ return records;
335
+ }
336
+
337
+ function describe(error: unknown): string {
338
+ return error instanceof Error ? error.message : String(error);
339
+ }