@rulvar/testing 1.28.0 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +51 -6
- package/dist/index.js +140 -29
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -161,30 +161,75 @@ declare function defaultRedact(value: string): string;
|
|
|
161
161
|
*/
|
|
162
162
|
declare function requestHash(req: ChatRequest): string;
|
|
163
163
|
/**
|
|
164
|
-
* Wraps live adapters for recording: every
|
|
165
|
-
*
|
|
166
|
-
*
|
|
164
|
+
* Wraps live adapters for recording: every stream that completes with
|
|
165
|
+
* exactly one terminal event (finish or error) appends one redacted
|
|
166
|
+
* row to the cassette JSONL. A stream that ends without a terminal
|
|
167
|
+
* (a requested abort or a truncated read), throws, or violates the
|
|
168
|
+
* adapter contract (a second terminal, data after the terminal)
|
|
169
|
+
* appends nothing, so a cassette row is always the record of one
|
|
170
|
+
* completed exchange (v1.28.0 review P2). The wrapped adapters are
|
|
171
|
+
* drop-in: same ids, providers, caps, and event streams.
|
|
167
172
|
*/
|
|
168
173
|
declare function record(options: {
|
|
169
174
|
adapters: ProviderAdapter[];
|
|
170
175
|
cassette: string;
|
|
171
176
|
redact?: RedactFn;
|
|
172
177
|
}): ProviderAdapter[];
|
|
173
|
-
/**
|
|
178
|
+
/**
|
|
179
|
+
* Typed hermetic-miss error; onMiss: 'throw' raises it on any request
|
|
180
|
+
* without a servable row. `recordedOccurrences` above zero means the
|
|
181
|
+
* hash WAS recorded but every occurrence is already consumed (replay
|
|
182
|
+
* serves each recorded exchange once, in file order); absent or zero
|
|
183
|
+
* means the request was never recorded at all (v1.29.0 review P2).
|
|
184
|
+
*/
|
|
174
185
|
declare class VcrMissError extends Error {
|
|
175
186
|
readonly requestHash: string;
|
|
176
|
-
|
|
187
|
+
/** Rows recorded for this hash; absent or 0 = never recorded. */
|
|
188
|
+
readonly recordedOccurrences?: number;
|
|
189
|
+
constructor(adapterId: string, hash: string, recordedOccurrences?: number);
|
|
177
190
|
}
|
|
178
191
|
interface VcrCassette {
|
|
179
192
|
header: VcrHeader;
|
|
180
193
|
rows: VcrRow[];
|
|
181
194
|
}
|
|
182
|
-
/**
|
|
195
|
+
/**
|
|
196
|
+
* Parses a cassette file (one header line plus one JSON row per line).
|
|
197
|
+
* The header must declare cassette format `v: 1`: the format version
|
|
198
|
+
* gates parsing itself, while hashVersion (whose support window is
|
|
199
|
+
* checked by replay) only gates request identity and never
|
|
200
|
+
* substitutes for it, so a future incompatible format refuses loudly
|
|
201
|
+
* instead of being read as v1. Every documented header field (kind,
|
|
202
|
+
* v, an integer hashVersion, a date-string recordedAt) and row field
|
|
203
|
+
* (adapterId, model, requestHash, request, caps, events, an optional
|
|
204
|
+
* string provider) is shape-checked here; unknown extra fields are
|
|
205
|
+
* tolerated for forward compatibility. Event stream SEMANTICS (one
|
|
206
|
+
* trailing terminal per row) are deliberately not checked at read
|
|
207
|
+
* time; `replay` enforces them before serving anything (v1.29.0
|
|
208
|
+
* review P3). Parse and shape failures throw a typed ConfigError
|
|
209
|
+
* naming the cassette path and line (v1.28.0 review P3).
|
|
210
|
+
*/
|
|
183
211
|
declare function readCassette(path: string): VcrCassette;
|
|
184
212
|
/**
|
|
185
213
|
* Builds replay adapters from a cassette. `onMiss: 'throw'` is the
|
|
186
214
|
* hermetic CI mode; `'passthrough'` forwards unrecorded requests to the
|
|
187
215
|
* matching live adapter in `adapters` (a development convenience only).
|
|
216
|
+
*
|
|
217
|
+
* Repeated hashes replay in file order (v1.29.0 review P2): rows
|
|
218
|
+
* sharing a `(adapterId, requestHash)` key form an ordered occurrence
|
|
219
|
+
* list, and every `stream()` call consumes exactly one occurrence,
|
|
220
|
+
* allocated synchronously inside the call itself, so two concurrent
|
|
221
|
+
* identical requests can never be served the same recorded exchange.
|
|
222
|
+
* A call after the last occurrence is a miss: under `onMiss: 'throw'`
|
|
223
|
+
* it raises a VcrMissError whose `recordedOccurrences` says the hash
|
|
224
|
+
* WAS recorded but is exhausted, and under `'passthrough'` it
|
|
225
|
+
* forwards to the live adapter exactly like a never-recorded request.
|
|
226
|
+
*
|
|
227
|
+
* Before serving anything, replay also enforces what `record` has
|
|
228
|
+
* guaranteed since v1.29.0: every row's event stream ends with
|
|
229
|
+
* exactly one terminal event (finish or error), and all caps
|
|
230
|
+
* snapshots for one `(adapterId, model)` agree, since the replay
|
|
231
|
+
* adapter can only report one caps truth per model. Violations throw
|
|
232
|
+
* a typed ConfigError naming the cassette and row.
|
|
188
233
|
*/
|
|
189
234
|
declare function replay(options: {
|
|
190
235
|
cassette: string;
|
package/dist/index.js
CHANGED
|
@@ -317,6 +317,10 @@ function requestHash(req) {
|
|
|
317
317
|
};
|
|
318
318
|
return createHash("sha256").update(canonicalJson(withoutTelemetry), "utf8").digest("hex");
|
|
319
319
|
}
|
|
320
|
+
/** The terminal vocabulary of the adapter contract: finish or error. */
|
|
321
|
+
function isTerminalEvent(event) {
|
|
322
|
+
return event.type === "finish" || event.type === "error";
|
|
323
|
+
}
|
|
320
324
|
function headerLine() {
|
|
321
325
|
return JSON.stringify({
|
|
322
326
|
v: 1,
|
|
@@ -326,9 +330,14 @@ function headerLine() {
|
|
|
326
330
|
});
|
|
327
331
|
}
|
|
328
332
|
/**
|
|
329
|
-
* Wraps live adapters for recording: every
|
|
330
|
-
*
|
|
331
|
-
*
|
|
333
|
+
* Wraps live adapters for recording: every stream that completes with
|
|
334
|
+
* exactly one terminal event (finish or error) appends one redacted
|
|
335
|
+
* row to the cassette JSONL. A stream that ends without a terminal
|
|
336
|
+
* (a requested abort or a truncated read), throws, or violates the
|
|
337
|
+
* adapter contract (a second terminal, data after the terminal)
|
|
338
|
+
* appends nothing, so a cassette row is always the record of one
|
|
339
|
+
* completed exchange (v1.28.0 review P2). The wrapped adapters are
|
|
340
|
+
* drop-in: same ids, providers, caps, and event streams.
|
|
332
341
|
*/
|
|
333
342
|
function record(options) {
|
|
334
343
|
const redact = options.redact ? (value) => defaultRedact(options.redact ? options.redact(value) : value) : defaultRedact;
|
|
@@ -341,8 +350,12 @@ function record(options) {
|
|
|
341
350
|
async *stream(req, signal) {
|
|
342
351
|
const events = [];
|
|
343
352
|
let thrown = false;
|
|
353
|
+
let terminals = 0;
|
|
354
|
+
let postTerminal = false;
|
|
344
355
|
try {
|
|
345
356
|
for await (const event of adapter.stream(req, signal)) {
|
|
357
|
+
if (terminals > 0) postTerminal = true;
|
|
358
|
+
if (isTerminalEvent(event)) terminals += 1;
|
|
346
359
|
events.push(event);
|
|
347
360
|
yield event;
|
|
348
361
|
}
|
|
@@ -350,7 +363,7 @@ function record(options) {
|
|
|
350
363
|
thrown = true;
|
|
351
364
|
throw error;
|
|
352
365
|
} finally {
|
|
353
|
-
if (!thrown) {
|
|
366
|
+
if (!thrown && terminals === 1 && !postTerminal) {
|
|
354
367
|
const row = {
|
|
355
368
|
adapterId: adapter.id,
|
|
356
369
|
...adapter.provider === void 0 ? {} : { provider: adapter.provider },
|
|
@@ -366,67 +379,165 @@ function record(options) {
|
|
|
366
379
|
}
|
|
367
380
|
}));
|
|
368
381
|
}
|
|
369
|
-
/**
|
|
382
|
+
/**
|
|
383
|
+
* Typed hermetic-miss error; onMiss: 'throw' raises it on any request
|
|
384
|
+
* without a servable row. `recordedOccurrences` above zero means the
|
|
385
|
+
* hash WAS recorded but every occurrence is already consumed (replay
|
|
386
|
+
* serves each recorded exchange once, in file order); absent or zero
|
|
387
|
+
* means the request was never recorded at all (v1.29.0 review P2).
|
|
388
|
+
*/
|
|
370
389
|
var VcrMissError = class extends Error {
|
|
371
390
|
requestHash;
|
|
372
|
-
|
|
373
|
-
|
|
391
|
+
/** Rows recorded for this hash; absent or 0 = never recorded. */
|
|
392
|
+
recordedOccurrences;
|
|
393
|
+
constructor(adapterId, hash, recordedOccurrences) {
|
|
394
|
+
super(recordedOccurrences !== void 0 && recordedOccurrences > 0 ? `VCR miss: adapter '${adapterId}' exhausted the ${String(recordedOccurrences)} recorded occurrence${recordedOccurrences === 1 ? "" : "s"} of request hash ${hash.slice(0, 12)}; a replay serves each recorded exchange once, in file order` : `VCR miss: adapter '${adapterId}' received a request with no recorded row (hash ${hash.slice(0, 12)}); onMiss: 'throw' keeps cassette tests hermetic`);
|
|
374
395
|
this.name = "VcrMissError";
|
|
375
396
|
this.requestHash = hash;
|
|
397
|
+
if (recordedOccurrences !== void 0) this.recordedOccurrences = recordedOccurrences;
|
|
376
398
|
}
|
|
377
399
|
};
|
|
378
|
-
/**
|
|
400
|
+
/**
|
|
401
|
+
* Parses a cassette file (one header line plus one JSON row per line).
|
|
402
|
+
* The header must declare cassette format `v: 1`: the format version
|
|
403
|
+
* gates parsing itself, while hashVersion (whose support window is
|
|
404
|
+
* checked by replay) only gates request identity and never
|
|
405
|
+
* substitutes for it, so a future incompatible format refuses loudly
|
|
406
|
+
* instead of being read as v1. Every documented header field (kind,
|
|
407
|
+
* v, an integer hashVersion, a date-string recordedAt) and row field
|
|
408
|
+
* (adapterId, model, requestHash, request, caps, events, an optional
|
|
409
|
+
* string provider) is shape-checked here; unknown extra fields are
|
|
410
|
+
* tolerated for forward compatibility. Event stream SEMANTICS (one
|
|
411
|
+
* trailing terminal per row) are deliberately not checked at read
|
|
412
|
+
* time; `replay` enforces them before serving anything (v1.29.0
|
|
413
|
+
* review P3). Parse and shape failures throw a typed ConfigError
|
|
414
|
+
* naming the cassette path and line (v1.28.0 review P3).
|
|
415
|
+
*/
|
|
379
416
|
function readCassette(path) {
|
|
380
|
-
const
|
|
381
|
-
|
|
382
|
-
|
|
417
|
+
const numbered = readFileSync(path, "utf8").split("\n").map((text, index) => ({
|
|
418
|
+
text,
|
|
419
|
+
lineNo: index + 1
|
|
420
|
+
})).filter(({ text }) => text.trim() !== "");
|
|
421
|
+
const parse = (line) => {
|
|
422
|
+
try {
|
|
423
|
+
return JSON.parse(line.text);
|
|
424
|
+
} catch {
|
|
425
|
+
throw new ConfigError(`${path}:${String(line.lineNo)} is not valid JSON; the cassette is corrupt or truncated`);
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
const first = numbered[0];
|
|
429
|
+
const headerRaw = first === void 0 ? {} : parse(first);
|
|
430
|
+
if (headerRaw.kind !== "rulvar-vcr") throw new ConfigError(`${path} is not a rulvar VCR cassette`);
|
|
431
|
+
if (headerRaw.v !== 1) throw new ConfigError(`${path} declares cassette format v ${String(headerRaw.v)}; this build reads format v 1 only, so record the cassette again on a matching engine`);
|
|
432
|
+
if (typeof headerRaw.hashVersion !== "number" || !Number.isSafeInteger(headerRaw.hashVersion)) throw new ConfigError(`${path} header hashVersion must be an integer (the identity profile version every recorded cassette carries); got ${String(headerRaw.hashVersion)}`);
|
|
433
|
+
if (typeof headerRaw.recordedAt !== "string" || Number.isNaN(Date.parse(headerRaw.recordedAt))) throw new ConfigError(`${path} header recordedAt must be a date string; got ${String(headerRaw.recordedAt)}`);
|
|
383
434
|
return {
|
|
384
|
-
header,
|
|
385
|
-
rows:
|
|
435
|
+
header: headerRaw,
|
|
436
|
+
rows: numbered.slice(1).map((line) => {
|
|
437
|
+
const parsed = parse(line);
|
|
438
|
+
const reject = (what) => {
|
|
439
|
+
throw new ConfigError(`${path}:${String(line.lineNo)} is not a VCR row: ${what}`);
|
|
440
|
+
};
|
|
441
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) reject("each row is a JSON object");
|
|
442
|
+
const row = parsed;
|
|
443
|
+
if (typeof row.adapterId !== "string" || row.adapterId === "") reject("adapterId must be a nonempty string");
|
|
444
|
+
if (typeof row.requestHash !== "string" || row.requestHash === "") reject("requestHash must be a nonempty string");
|
|
445
|
+
if (typeof row.model !== "string" || row.model === "") reject("model must be a nonempty string");
|
|
446
|
+
if (row.provider !== void 0 && typeof row.provider !== "string") reject("provider, when present, must be a string");
|
|
447
|
+
if (typeof row.request !== "object" || row.request === null) reject("request must be an object (the redacted canonical request)");
|
|
448
|
+
if (typeof row.caps !== "object" || row.caps === null || Array.isArray(row.caps)) reject("caps must be an object (the model caps snapshot at record time)");
|
|
449
|
+
if (!Array.isArray(row.events)) reject("events must be an array");
|
|
450
|
+
return row;
|
|
451
|
+
})
|
|
386
452
|
};
|
|
387
453
|
}
|
|
388
454
|
/**
|
|
389
455
|
* Builds replay adapters from a cassette. `onMiss: 'throw'` is the
|
|
390
456
|
* hermetic CI mode; `'passthrough'` forwards unrecorded requests to the
|
|
391
457
|
* matching live adapter in `adapters` (a development convenience only).
|
|
458
|
+
*
|
|
459
|
+
* Repeated hashes replay in file order (v1.29.0 review P2): rows
|
|
460
|
+
* sharing a `(adapterId, requestHash)` key form an ordered occurrence
|
|
461
|
+
* list, and every `stream()` call consumes exactly one occurrence,
|
|
462
|
+
* allocated synchronously inside the call itself, so two concurrent
|
|
463
|
+
* identical requests can never be served the same recorded exchange.
|
|
464
|
+
* A call after the last occurrence is a miss: under `onMiss: 'throw'`
|
|
465
|
+
* it raises a VcrMissError whose `recordedOccurrences` says the hash
|
|
466
|
+
* WAS recorded but is exhausted, and under `'passthrough'` it
|
|
467
|
+
* forwards to the live adapter exactly like a never-recorded request.
|
|
468
|
+
*
|
|
469
|
+
* Before serving anything, replay also enforces what `record` has
|
|
470
|
+
* guaranteed since v1.29.0: every row's event stream ends with
|
|
471
|
+
* exactly one terminal event (finish or error), and all caps
|
|
472
|
+
* snapshots for one `(adapterId, model)` agree, since the replay
|
|
473
|
+
* adapter can only report one caps truth per model. Violations throw
|
|
474
|
+
* a typed ConfigError naming the cassette and row.
|
|
392
475
|
*/
|
|
393
476
|
function replay(options) {
|
|
394
477
|
const { header, rows } = readCassette(options.cassette);
|
|
395
478
|
const oldestSupported = CURRENT_HASH_VERSION - 1;
|
|
396
|
-
if (
|
|
479
|
+
if (header.hashVersion < oldestSupported || header.hashVersion > CURRENT_HASH_VERSION) throw new ConfigError(`${options.cassette} was recorded under hashVersion ${String(header.hashVersion)}, outside the supported window [${oldestSupported}, ${CURRENT_HASH_VERSION}]; record the cassette again on a current engine`);
|
|
480
|
+
rows.forEach((row, index) => {
|
|
481
|
+
const terminals = row.events.filter((event) => isTerminalEvent(event)).length;
|
|
482
|
+
const last = row.events[row.events.length - 1];
|
|
483
|
+
if (terminals !== 1 || last === void 0 || !isTerminalEvent(last)) throw new ConfigError(`${options.cassette} row ${String(index + 1)} (adapter '${row.adapterId}', hash ${row.requestHash.slice(0, 12)}) does not record one completed exchange: expected exactly one trailing terminal event (finish or error), found ${String(terminals)}; record the cassette again on a current engine`);
|
|
484
|
+
});
|
|
397
485
|
const byAdapter = /* @__PURE__ */ new Map();
|
|
398
486
|
for (const row of rows) {
|
|
399
487
|
const forAdapter = byAdapter.get(row.adapterId) ?? /* @__PURE__ */ new Map();
|
|
400
|
-
forAdapter.
|
|
488
|
+
const occurrences = forAdapter.get(row.requestHash) ?? [];
|
|
489
|
+
occurrences.push(row);
|
|
490
|
+
forAdapter.set(row.requestHash, occurrences);
|
|
401
491
|
byAdapter.set(row.adapterId, forAdapter);
|
|
402
492
|
}
|
|
403
493
|
const live = new Map((options.adapters ?? []).map((adapter) => [adapter.id, adapter]));
|
|
404
494
|
return [.../* @__PURE__ */ new Set([...byAdapter.keys(), ...live.keys()])].map((adapterId) => {
|
|
405
495
|
const recorded = byAdapter.get(adapterId) ?? /* @__PURE__ */ new Map();
|
|
406
|
-
const
|
|
407
|
-
const someRow = [
|
|
496
|
+
const recordedRows = [...recorded.values()].flat();
|
|
497
|
+
const someRow = recordedRows[0];
|
|
408
498
|
const capsByModel = /* @__PURE__ */ new Map();
|
|
409
|
-
for (const row of
|
|
499
|
+
for (const row of recordedRows) {
|
|
500
|
+
const canonical = canonicalJson(row.caps);
|
|
501
|
+
const existing = capsByModel.get(row.model);
|
|
502
|
+
if (existing === void 0) capsByModel.set(row.model, {
|
|
503
|
+
caps: row.caps,
|
|
504
|
+
canonical
|
|
505
|
+
});
|
|
506
|
+
else if (existing.canonical !== canonical) throw new ConfigError(`${options.cassette} carries conflicting caps snapshots for adapter '${adapterId}' model '${row.model}'; a replay adapter reports one caps truth per model, so record the cassette again in one session`);
|
|
507
|
+
}
|
|
508
|
+
const passthrough = live.get(adapterId);
|
|
509
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
410
510
|
return {
|
|
411
511
|
id: adapterId,
|
|
412
512
|
...someRow?.provider === void 0 ? {} : { provider: someRow.provider },
|
|
413
513
|
caps: (model) => {
|
|
414
|
-
const snapshot = capsByModel.get(model) ?? passthrough?.caps(model);
|
|
514
|
+
const snapshot = capsByModel.get(model)?.caps ?? passthrough?.caps(model);
|
|
415
515
|
if (snapshot === void 0) throw new ConfigError(`VCR replay adapter '${adapterId}' has no caps snapshot for model '${model}'`);
|
|
416
516
|
return snapshot;
|
|
417
517
|
},
|
|
418
|
-
|
|
518
|
+
stream(req, signal) {
|
|
419
519
|
const hash = requestHash(req);
|
|
420
|
-
const
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
520
|
+
const occurrences = recorded.get(hash);
|
|
521
|
+
let row;
|
|
522
|
+
if (occurrences !== void 0) {
|
|
523
|
+
const cursor = cursors.get(hash) ?? 0;
|
|
524
|
+
if (cursor < occurrences.length) {
|
|
525
|
+
cursors.set(hash, cursor + 1);
|
|
526
|
+
row = occurrences[cursor];
|
|
527
|
+
}
|
|
428
528
|
}
|
|
429
|
-
|
|
529
|
+
const recordedCount = occurrences?.length;
|
|
530
|
+
return (async function* () {
|
|
531
|
+
if (row !== void 0) {
|
|
532
|
+
for (const event of row.events) yield event;
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
if (options.onMiss === "passthrough" && passthrough !== void 0) {
|
|
536
|
+
yield* passthrough.stream(req, signal);
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
throw new VcrMissError(adapterId, hash, recordedCount);
|
|
540
|
+
})();
|
|
430
541
|
}
|
|
431
542
|
};
|
|
432
543
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulvar/testing",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.30.0",
|
|
4
4
|
"description": "Rulvar test harness: createTestEngine, FakeAdapter, VCR cassettes, replay-strict runs, matchers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"access": "public"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@rulvar/core": "1.
|
|
29
|
+
"@rulvar/core": "1.30.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node": "^22.20.0",
|