@rulvar/testing 1.30.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +76 -16
- package/dist/index.js +268 -55
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -130,6 +130,29 @@ declare function runLiveSmoke(adapter: Pick<ProviderAdapter, "stream">, req: Cha
|
|
|
130
130
|
interface VcrRow {
|
|
131
131
|
adapterId: string;
|
|
132
132
|
provider?: string;
|
|
133
|
+
/**
|
|
134
|
+
* The recording adapter's declared usageSemantics snapshot (v1.30.0
|
|
135
|
+
* review P2): replay restores it on the rebuilt adapter, so the
|
|
136
|
+
* fresh journal of a replayed run carries the same provenance stamp
|
|
137
|
+
* the recorded run got. Absent when the recording adapter declared
|
|
138
|
+
* none, and in every cassette recorded before v1.31.0, whose
|
|
139
|
+
* replays therefore stamp nothing (documented historical laxity; an
|
|
140
|
+
* unstamped entry reads as recorded before the stamp existed).
|
|
141
|
+
*/
|
|
142
|
+
usageSemantics?: string;
|
|
143
|
+
/**
|
|
144
|
+
* Zero based per `(adapterId, requestHash)` call counter, claimed
|
|
145
|
+
* synchronously when the recorded `stream()` call was made
|
|
146
|
+
* (v1.31.0 review P2): rows are appended in COMPLETION order, so
|
|
147
|
+
* without this number two concurrent identical live calls that
|
|
148
|
+
* finish out of order would swap callers at replay, which hands
|
|
149
|
+
* occurrences out in caller order. Replay sorts same hash rows by
|
|
150
|
+
* it when every row of the group carries one; absent in cassettes
|
|
151
|
+
* recorded before v1.32.0, whose same hash rows keep file order.
|
|
152
|
+
* An aborted or failed call claims a number but appends no row, so
|
|
153
|
+
* gaps in the numbering are valid.
|
|
154
|
+
*/
|
|
155
|
+
occurrence?: number;
|
|
133
156
|
requestHash: string;
|
|
134
157
|
/** Redacted canonical request, for humans and drift review. */
|
|
135
158
|
request: unknown;
|
|
@@ -167,8 +190,13 @@ declare function requestHash(req: ChatRequest): string;
|
|
|
167
190
|
* (a requested abort or a truncated read), throws, or violates the
|
|
168
191
|
* adapter contract (a second terminal, data after the terminal)
|
|
169
192
|
* appends nothing, so a cassette row is always the record of one
|
|
170
|
-
* completed exchange (v1.28.0 review P2).
|
|
171
|
-
*
|
|
193
|
+
* completed exchange (v1.28.0 review P2). Every call also claims a
|
|
194
|
+
* per `(adapterId, requestHash)` occurrence number synchronously in
|
|
195
|
+
* the `stream()` call itself and persists it on the completed row,
|
|
196
|
+
* so replay can restore the caller to response association even when
|
|
197
|
+
* concurrent identical calls completed out of order (v1.31.0 review
|
|
198
|
+
* P2). The wrapped adapters are drop-in: same ids, providers, caps,
|
|
199
|
+
* and event streams.
|
|
172
200
|
*/
|
|
173
201
|
declare function record(options: {
|
|
174
202
|
adapters: ProviderAdapter[];
|
|
@@ -179,8 +207,9 @@ declare function record(options: {
|
|
|
179
207
|
* Typed hermetic-miss error; onMiss: 'throw' raises it on any request
|
|
180
208
|
* without a servable row. `recordedOccurrences` above zero means the
|
|
181
209
|
* hash WAS recorded but every occurrence is already consumed (replay
|
|
182
|
-
* serves each recorded exchange once, in
|
|
183
|
-
* means the request was never recorded at all (v1.29.0 review
|
|
210
|
+
* serves each recorded exchange once, in recorded order); absent or
|
|
211
|
+
* zero means the request was never recorded at all (v1.29.0 review
|
|
212
|
+
* P2).
|
|
184
213
|
*/
|
|
185
214
|
declare class VcrMissError extends Error {
|
|
186
215
|
readonly requestHash: string;
|
|
@@ -199,14 +228,23 @@ interface VcrCassette {
|
|
|
199
228
|
* checked by replay) only gates request identity and never
|
|
200
229
|
* substitutes for it, so a future incompatible format refuses loudly
|
|
201
230
|
* instead of being read as v1. Every documented header field (kind,
|
|
202
|
-
* v, an integer hashVersion, a date
|
|
231
|
+
* v, an integer hashVersion, a date string recordedAt) and row field
|
|
203
232
|
* (adapterId, model, requestHash, request, caps, events, an optional
|
|
204
|
-
* string provider
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
233
|
+
* string provider, an optional nonempty usageSemantics, an optional
|
|
234
|
+
* nonnegative integer occurrence) is checked
|
|
235
|
+
* here, and the nested structures are validated in depth (v1.30.0
|
|
236
|
+
* review P3): the request must be a plain object, every event must
|
|
237
|
+
* be a member of the canonical ChatEvent vocabulary with its
|
|
238
|
+
* required payload and Usage numeric invariants, and caps must carry
|
|
239
|
+
* every ModelCaps field (with the optional pricing table checked
|
|
240
|
+
* when present). Unknown extra FIELDS are tolerated for forward
|
|
241
|
+
* compatibility. Event stream SEMANTICS (one trailing terminal per
|
|
242
|
+
* row) and adapter consistency across rows (provider,
|
|
243
|
+
* usageSemantics, caps agreement) are deliberately not checked at
|
|
244
|
+
* read time; `replay` enforces them before serving anything (v1.29.0
|
|
245
|
+
* review P3), so reading never blocks inspecting a well formed file.
|
|
246
|
+
* Parse and shape failures throw a typed ConfigError naming the
|
|
247
|
+
* cassette path and line (v1.28.0 review P3).
|
|
210
248
|
*/
|
|
211
249
|
declare function readCassette(path: string): VcrCassette;
|
|
212
250
|
/**
|
|
@@ -214,11 +252,17 @@ declare function readCassette(path: string): VcrCassette;
|
|
|
214
252
|
* hermetic CI mode; `'passthrough'` forwards unrecorded requests to the
|
|
215
253
|
* matching live adapter in `adapters` (a development convenience only).
|
|
216
254
|
*
|
|
217
|
-
* Repeated hashes replay
|
|
218
|
-
* sharing a `(adapterId, requestHash)` key form an ordered
|
|
219
|
-
* list, and every `stream()` call consumes exactly one
|
|
220
|
-
* allocated synchronously inside the call itself, so two
|
|
221
|
-
* identical requests can never be served the same
|
|
255
|
+
* Repeated hashes replay as ordered occurrences (v1.29.0 review P2):
|
|
256
|
+
* rows sharing a `(adapterId, requestHash)` key form an ordered
|
|
257
|
+
* occurrence list, and every `stream()` call consumes exactly one
|
|
258
|
+
* occurrence, allocated synchronously inside the call itself, so two
|
|
259
|
+
* concurrent identical requests can never be served the same
|
|
260
|
+
* recorded exchange. The list is sorted by the recorded `occurrence`
|
|
261
|
+
* numbers when every row of the group carries one, so concurrent
|
|
262
|
+
* identical calls whose live completions were appended out of order
|
|
263
|
+
* still replay to the callers that made them (v1.31.0 review P2); a
|
|
264
|
+
* group with any unnumbered row (recorded before v1.32.0) keeps file
|
|
265
|
+
* order.
|
|
222
266
|
* A call after the last occurrence is a miss: under `onMiss: 'throw'`
|
|
223
267
|
* it raises a VcrMissError whose `recordedOccurrences` says the hash
|
|
224
268
|
* WAS recorded but is exhausted, and under `'passthrough'` it
|
|
@@ -230,6 +274,22 @@ declare function readCassette(path: string): VcrCassette;
|
|
|
230
274
|
* snapshots for one `(adapterId, model)` agree, since the replay
|
|
231
275
|
* adapter can only report one caps truth per model. Violations throw
|
|
232
276
|
* a typed ConfigError naming the cassette and row.
|
|
277
|
+
*
|
|
278
|
+
* The rebuilt adapter restores the recorded provider and
|
|
279
|
+
* usageSemantics declarations (v1.30.0 review P2), so the fresh
|
|
280
|
+
* journal of a replayed run carries the same provenance stamp the
|
|
281
|
+
* recorded run got instead of silently reading like an entry from
|
|
282
|
+
* before the stamp existed. All rows of one adapter must agree on
|
|
283
|
+
* both declarations; a conflict refuses with a typed ConfigError
|
|
284
|
+
* before anything is served. A cassette recorded before v1.31.0
|
|
285
|
+
* stores no usageSemantics, so its replays stamp nothing (documented
|
|
286
|
+
* historical laxity). Under `onMiss: 'passthrough'` the recorded
|
|
287
|
+
* declarations must also match the live adapter's, absent versus
|
|
288
|
+
* present included, because a live served miss is journaled under
|
|
289
|
+
* the wrapper's declarations; a mismatch refuses at construction
|
|
290
|
+
* (v1.31.0 review P2). An adapter with no recorded rows keeps the
|
|
291
|
+
* live adapter's own declarations, so the wrapper stays a metadata
|
|
292
|
+
* preserving drop in.
|
|
233
293
|
*/
|
|
234
294
|
declare function replay(options: {
|
|
235
295
|
cassette: string;
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { a as fakeWireError, i as fakeToolCalls, n as FAKE_MODEL_REF, r as FakeAdapter, t as FAKE_MODEL } from "./fake-adapter-3T_w-IOY.js";
|
|
2
|
-
import { CURRENT_HASH_VERSION, ConfigError, InMemoryStore, JournalMissError, createEngine, hashWorkflowBody } from "@rulvar/core";
|
|
2
|
+
import { CURRENT_HASH_VERSION, ConfigError, InMemoryStore, JournalMissError, createEngine, hashWorkflowBody, usageViolations } from "@rulvar/core";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
5
5
|
//#region src/test-engine.ts
|
|
@@ -336,67 +336,227 @@ function headerLine() {
|
|
|
336
336
|
* (a requested abort or a truncated read), throws, or violates the
|
|
337
337
|
* adapter contract (a second terminal, data after the terminal)
|
|
338
338
|
* appends nothing, so a cassette row is always the record of one
|
|
339
|
-
* completed exchange (v1.28.0 review P2).
|
|
340
|
-
*
|
|
339
|
+
* completed exchange (v1.28.0 review P2). Every call also claims a
|
|
340
|
+
* per `(adapterId, requestHash)` occurrence number synchronously in
|
|
341
|
+
* the `stream()` call itself and persists it on the completed row,
|
|
342
|
+
* so replay can restore the caller to response association even when
|
|
343
|
+
* concurrent identical calls completed out of order (v1.31.0 review
|
|
344
|
+
* P2). The wrapped adapters are drop-in: same ids, providers, caps,
|
|
345
|
+
* and event streams.
|
|
341
346
|
*/
|
|
342
347
|
function record(options) {
|
|
343
348
|
const redact = options.redact ? (value) => defaultRedact(options.redact ? options.redact(value) : value) : defaultRedact;
|
|
344
349
|
if (!existsSync(options.cassette)) writeFileSync(options.cassette, `${headerLine()}\n`, "utf8");
|
|
345
|
-
return options.adapters.map((adapter) =>
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
350
|
+
return options.adapters.map((adapter) => {
|
|
351
|
+
const occurrences = /* @__PURE__ */ new Map();
|
|
352
|
+
return {
|
|
353
|
+
...adapter,
|
|
354
|
+
id: adapter.id,
|
|
355
|
+
...adapter.provider === void 0 ? {} : { provider: adapter.provider },
|
|
356
|
+
caps: (model) => adapter.caps(model),
|
|
357
|
+
stream(req, signal) {
|
|
358
|
+
const hash = requestHash(req);
|
|
359
|
+
const occurrence = occurrences.get(hash) ?? 0;
|
|
360
|
+
occurrences.set(hash, occurrence + 1);
|
|
361
|
+
return (async function* () {
|
|
362
|
+
const events = [];
|
|
363
|
+
let thrown = false;
|
|
364
|
+
let terminals = 0;
|
|
365
|
+
let postTerminal = false;
|
|
366
|
+
try {
|
|
367
|
+
for await (const event of adapter.stream(req, signal)) {
|
|
368
|
+
if (terminals > 0) postTerminal = true;
|
|
369
|
+
if (isTerminalEvent(event)) terminals += 1;
|
|
370
|
+
events.push(event);
|
|
371
|
+
yield event;
|
|
372
|
+
}
|
|
373
|
+
} catch (error) {
|
|
374
|
+
thrown = true;
|
|
375
|
+
throw error;
|
|
376
|
+
} finally {
|
|
377
|
+
if (!thrown && terminals === 1 && !postTerminal) {
|
|
378
|
+
const row = {
|
|
379
|
+
adapterId: adapter.id,
|
|
380
|
+
...adapter.provider === void 0 ? {} : { provider: adapter.provider },
|
|
381
|
+
...adapter.usageSemantics === void 0 ? {} : { usageSemantics: adapter.usageSemantics },
|
|
382
|
+
occurrence,
|
|
383
|
+
requestHash: hash,
|
|
384
|
+
request: walkStrings(JSON.parse(JSON.stringify(req)), redact),
|
|
385
|
+
events: walkStrings(JSON.parse(JSON.stringify(events)), redact),
|
|
386
|
+
caps: adapter.caps(req.model),
|
|
387
|
+
model: req.model
|
|
388
|
+
};
|
|
389
|
+
appendFileSync(options.cassette, `${JSON.stringify(row)}\n`, "utf8");
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
})();
|
|
378
393
|
}
|
|
379
|
-
}
|
|
380
|
-
})
|
|
394
|
+
};
|
|
395
|
+
});
|
|
381
396
|
}
|
|
382
397
|
/**
|
|
383
398
|
* Typed hermetic-miss error; onMiss: 'throw' raises it on any request
|
|
384
399
|
* without a servable row. `recordedOccurrences` above zero means the
|
|
385
400
|
* hash WAS recorded but every occurrence is already consumed (replay
|
|
386
|
-
* serves each recorded exchange once, in
|
|
387
|
-
* means the request was never recorded at all (v1.29.0 review
|
|
401
|
+
* serves each recorded exchange once, in recorded order); absent or
|
|
402
|
+
* zero means the request was never recorded at all (v1.29.0 review
|
|
403
|
+
* P2).
|
|
388
404
|
*/
|
|
389
405
|
var VcrMissError = class extends Error {
|
|
390
406
|
requestHash;
|
|
391
407
|
/** Rows recorded for this hash; absent or 0 = never recorded. */
|
|
392
408
|
recordedOccurrences;
|
|
393
409
|
constructor(adapterId, hash, recordedOccurrences) {
|
|
394
|
-
super(recordedOccurrences !== void 0 && recordedOccurrences > 0 ? `VCR miss: adapter '${adapterId}' exhausted the ${String(recordedOccurrences)} recorded occurrence${recordedOccurrences === 1 ? "" : "s"} of request hash ${hash.slice(0, 12)}; a replay serves each recorded exchange once, in
|
|
410
|
+
super(recordedOccurrences !== void 0 && recordedOccurrences > 0 ? `VCR miss: adapter '${adapterId}' exhausted the ${String(recordedOccurrences)} recorded occurrence${recordedOccurrences === 1 ? "" : "s"} of request hash ${hash.slice(0, 12)}; a replay serves each recorded exchange once, in recorded order` : `VCR miss: adapter '${adapterId}' received a request with no recorded row (hash ${hash.slice(0, 12)}); onMiss: 'throw' keeps cassette tests hermetic`);
|
|
395
411
|
this.name = "VcrMissError";
|
|
396
412
|
this.requestHash = hash;
|
|
397
413
|
if (recordedOccurrences !== void 0) this.recordedOccurrences = recordedOccurrences;
|
|
398
414
|
}
|
|
399
415
|
};
|
|
416
|
+
function isPlainObject(value) {
|
|
417
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
418
|
+
}
|
|
419
|
+
const EFFORTS = [
|
|
420
|
+
"low",
|
|
421
|
+
"medium",
|
|
422
|
+
"high",
|
|
423
|
+
"xhigh",
|
|
424
|
+
"max"
|
|
425
|
+
];
|
|
426
|
+
const FINISH_REASONS = [
|
|
427
|
+
"stop",
|
|
428
|
+
"tool-calls",
|
|
429
|
+
"max-tokens",
|
|
430
|
+
"context-window-exceeded",
|
|
431
|
+
"refusal"
|
|
432
|
+
];
|
|
433
|
+
const USAGE_COUNT_FIELDS = [
|
|
434
|
+
"inputTokens",
|
|
435
|
+
"outputTokens",
|
|
436
|
+
"cacheReadTokens",
|
|
437
|
+
"cacheWriteTokens",
|
|
438
|
+
"reasoningTokens"
|
|
439
|
+
];
|
|
440
|
+
/**
|
|
441
|
+
* First shape violation of a recorded caps snapshot, or undefined. The
|
|
442
|
+
* checks mirror what ModelCaps declares (v1.30.0 review P3): before
|
|
443
|
+
* this shipped an empty object passed as a snapshot and the failure
|
|
444
|
+
* surfaced later as an unrelated router or pricing defect.
|
|
445
|
+
*/
|
|
446
|
+
function capsShapeIssue(caps) {
|
|
447
|
+
if (caps.structuredOutput !== "native" && caps.structuredOutput !== "forced-tool" && caps.structuredOutput !== "prompt") return "caps.structuredOutput must be 'native', 'forced-tool', or 'prompt'";
|
|
448
|
+
for (const field of ["supportsTemperature", "supportsParallelTools"]) if (typeof caps[field] !== "boolean") return `caps.${field} must be a boolean`;
|
|
449
|
+
const efforts = caps.reasoningEfforts;
|
|
450
|
+
if (!Array.isArray(efforts) || efforts.some((entry) => typeof entry !== "string" || !EFFORTS.includes(entry))) return "caps.reasoningEfforts must be an array of canonical efforts";
|
|
451
|
+
for (const field of ["contextWindow", "maxOutputTokens"]) {
|
|
452
|
+
const value = caps[field];
|
|
453
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) return `caps.${field} must be a positive safe integer`;
|
|
454
|
+
}
|
|
455
|
+
if (caps.pricing !== void 0) {
|
|
456
|
+
const pricing = caps.pricing;
|
|
457
|
+
if (!isPlainObject(pricing)) return "caps.pricing must be an object when present";
|
|
458
|
+
for (const field of ["inputUsdPerMTok", "outputUsdPerMTok"]) {
|
|
459
|
+
const value = pricing[field];
|
|
460
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return `caps.pricing.${field} must be a nonnegative finite number`;
|
|
461
|
+
}
|
|
462
|
+
for (const field of [
|
|
463
|
+
"cacheReadUsdPerMTok",
|
|
464
|
+
"cacheWriteUsdPerMTok",
|
|
465
|
+
"cacheWrite1hUsdPerMTok"
|
|
466
|
+
]) {
|
|
467
|
+
const value = pricing[field];
|
|
468
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || value < 0)) return `caps.pricing.${field} must be a nonnegative finite number when present`;
|
|
469
|
+
}
|
|
470
|
+
const tiers = pricing.tiers;
|
|
471
|
+
if (tiers !== void 0) {
|
|
472
|
+
if (!Array.isArray(tiers)) return "caps.pricing.tiers must be an array when present";
|
|
473
|
+
for (const [index, tier] of tiers.entries()) {
|
|
474
|
+
const path = `caps.pricing.tiers[${String(index)}]`;
|
|
475
|
+
if (!isPlainObject(tier)) return `${path} must be an object`;
|
|
476
|
+
const above = tier.aboveInputTokens;
|
|
477
|
+
if (typeof above !== "number" || !Number.isSafeInteger(above) || above < 0) return `${path}.aboveInputTokens must be a nonnegative safe integer`;
|
|
478
|
+
for (const field of ["inputMultiplier", "outputMultiplier"]) {
|
|
479
|
+
const value = tier[field];
|
|
480
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return `${path}.${field} must be a positive finite number`;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* First shape violation of one recorded event, or undefined. Every
|
|
488
|
+
* element must be a member of the canonical ChatEvent vocabulary with
|
|
489
|
+
* its required payload (v1.30.0 review P3): before this shipped a
|
|
490
|
+
* null element crashed replay with a raw TypeError and a bare
|
|
491
|
+
* `{ type: 'finish' }` reached the engine, which then died on the
|
|
492
|
+
* missing usage instead of refusing the cassette at its boundary.
|
|
493
|
+
* Unknown extra FIELDS on a known event stay tolerated; an unknown
|
|
494
|
+
* event TYPE is refused, because replay would feed it to an engine
|
|
495
|
+
* whose vocabulary provably does not include it (the cassette format
|
|
496
|
+
* version, not leniency here, is the growth path).
|
|
497
|
+
*/
|
|
498
|
+
function eventShapeIssue(event, index) {
|
|
499
|
+
const at = `events[${String(index)}]`;
|
|
500
|
+
if (!isPlainObject(event)) return `${at} must be an object (a canonical ChatEvent)`;
|
|
501
|
+
const type = event.type;
|
|
502
|
+
switch (type) {
|
|
503
|
+
case "text-delta":
|
|
504
|
+
case "reasoning-delta": return typeof event.text === "string" ? void 0 : `${at}.text must be a string`;
|
|
505
|
+
case "tool-call-start":
|
|
506
|
+
if (typeof event.id !== "string" || event.id === "") return `${at}.id must be a nonempty string`;
|
|
507
|
+
return typeof event.name === "string" && event.name !== "" ? void 0 : `${at}.name must be a nonempty string`;
|
|
508
|
+
case "tool-call-delta":
|
|
509
|
+
if (typeof event.id !== "string" || event.id === "") return `${at}.id must be a nonempty string`;
|
|
510
|
+
return typeof event.argsTextDelta === "string" ? void 0 : `${at}.argsTextDelta must be a string`;
|
|
511
|
+
case "tool-call-end":
|
|
512
|
+
if (typeof event.id !== "string" || event.id === "") return `${at}.id must be a nonempty string`;
|
|
513
|
+
return Object.hasOwn(event, "args") ? void 0 : `${at}.args must be present (the arguments the call ended with)`;
|
|
514
|
+
case "usage": {
|
|
515
|
+
const usage = event.usage;
|
|
516
|
+
if (!isPlainObject(usage)) return `${at}.usage must be an object`;
|
|
517
|
+
for (const field of USAGE_COUNT_FIELDS) {
|
|
518
|
+
const value = usage[field];
|
|
519
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)) return `${at}.usage.${field} must be a nonnegative safe integer when present`;
|
|
520
|
+
}
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
case "finish": {
|
|
524
|
+
const finish = event.finish;
|
|
525
|
+
if (!isPlainObject(finish)) return `${at}.finish must be an object (a typed FinishInfo)`;
|
|
526
|
+
const reason = finish.reason;
|
|
527
|
+
if (typeof reason !== "string" || !FINISH_REASONS.includes(reason)) return `${at}.finish.reason must be a canonical finish reason`;
|
|
528
|
+
if (reason === "refusal") {
|
|
529
|
+
const refusal = finish.refusal;
|
|
530
|
+
if (!isPlainObject(refusal) || typeof refusal.provider !== "string") return `${at}.finish.refusal must be an object naming the provider`;
|
|
531
|
+
const stopDetails = refusal.stopDetails;
|
|
532
|
+
if (stopDetails !== void 0) {
|
|
533
|
+
if (!isPlainObject(stopDetails)) return `${at}.finish.refusal.stopDetails must be an object when present`;
|
|
534
|
+
for (const field of [
|
|
535
|
+
"type",
|
|
536
|
+
"category",
|
|
537
|
+
"explanation"
|
|
538
|
+
]) {
|
|
539
|
+
const value = stopDetails[field];
|
|
540
|
+
if (value !== void 0 && typeof value !== "string") return `${at}.finish.refusal.stopDetails.${field} must be a string when present`;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
if (event.providerMetadata !== void 0 && !isPlainObject(event.providerMetadata)) return `${at}.providerMetadata must be a plain object when present`;
|
|
545
|
+
const usage = event.usage;
|
|
546
|
+
if (!isPlainObject(usage)) return `${at}.usage must be an object (the full Usage of the exchange)`;
|
|
547
|
+
const violations = usageViolations(usage);
|
|
548
|
+
return violations.length === 0 ? void 0 : `${at}.usage violates the Usage invariant: ${violations.join("; ")}`;
|
|
549
|
+
}
|
|
550
|
+
case "error": {
|
|
551
|
+
const error = event.error;
|
|
552
|
+
if (!isPlainObject(error)) return `${at}.error must be an object (a WireError)`;
|
|
553
|
+
if (typeof error.code !== "string" || error.code === "") return `${at}.error.code must be a nonempty string`;
|
|
554
|
+
if (typeof error.message !== "string") return `${at}.error.message must be a string`;
|
|
555
|
+
return typeof error.retryable === "boolean" ? void 0 : `${at}.error.retryable must be a boolean`;
|
|
556
|
+
}
|
|
557
|
+
default: return `${at}.type must be a canonical ChatEvent type; got ${typeof type === "string" ? `'${type}'` : String(type)}`;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
400
560
|
/**
|
|
401
561
|
* Parses a cassette file (one header line plus one JSON row per line).
|
|
402
562
|
* The header must declare cassette format `v: 1`: the format version
|
|
@@ -404,14 +564,23 @@ var VcrMissError = class extends Error {
|
|
|
404
564
|
* checked by replay) only gates request identity and never
|
|
405
565
|
* substitutes for it, so a future incompatible format refuses loudly
|
|
406
566
|
* instead of being read as v1. Every documented header field (kind,
|
|
407
|
-
* v, an integer hashVersion, a date
|
|
567
|
+
* v, an integer hashVersion, a date string recordedAt) and row field
|
|
408
568
|
* (adapterId, model, requestHash, request, caps, events, an optional
|
|
409
|
-
* string provider
|
|
410
|
-
*
|
|
411
|
-
*
|
|
412
|
-
*
|
|
413
|
-
*
|
|
414
|
-
*
|
|
569
|
+
* string provider, an optional nonempty usageSemantics, an optional
|
|
570
|
+
* nonnegative integer occurrence) is checked
|
|
571
|
+
* here, and the nested structures are validated in depth (v1.30.0
|
|
572
|
+
* review P3): the request must be a plain object, every event must
|
|
573
|
+
* be a member of the canonical ChatEvent vocabulary with its
|
|
574
|
+
* required payload and Usage numeric invariants, and caps must carry
|
|
575
|
+
* every ModelCaps field (with the optional pricing table checked
|
|
576
|
+
* when present). Unknown extra FIELDS are tolerated for forward
|
|
577
|
+
* compatibility. Event stream SEMANTICS (one trailing terminal per
|
|
578
|
+
* row) and adapter consistency across rows (provider,
|
|
579
|
+
* usageSemantics, caps agreement) are deliberately not checked at
|
|
580
|
+
* read time; `replay` enforces them before serving anything (v1.29.0
|
|
581
|
+
* review P3), so reading never blocks inspecting a well formed file.
|
|
582
|
+
* Parse and shape failures throw a typed ConfigError naming the
|
|
583
|
+
* cassette path and line (v1.28.0 review P3).
|
|
415
584
|
*/
|
|
416
585
|
function readCassette(path) {
|
|
417
586
|
const numbered = readFileSync(path, "utf8").split("\n").map((text, index) => ({
|
|
@@ -444,9 +613,18 @@ function readCassette(path) {
|
|
|
444
613
|
if (typeof row.requestHash !== "string" || row.requestHash === "") reject("requestHash must be a nonempty string");
|
|
445
614
|
if (typeof row.model !== "string" || row.model === "") reject("model must be a nonempty string");
|
|
446
615
|
if (row.provider !== void 0 && typeof row.provider !== "string") reject("provider, when present, must be a string");
|
|
447
|
-
if (typeof row.
|
|
616
|
+
if (row.usageSemantics !== void 0 && (typeof row.usageSemantics !== "string" || row.usageSemantics === "")) reject("usageSemantics, when present, must be a nonempty string");
|
|
617
|
+
if (row.occurrence !== void 0 && (typeof row.occurrence !== "number" || !Number.isSafeInteger(row.occurrence) || row.occurrence < 0)) reject("occurrence, when present, must be a nonnegative safe integer");
|
|
618
|
+
if (typeof row.request !== "object" || row.request === null || Array.isArray(row.request)) reject("request must be an object (the redacted canonical request)");
|
|
448
619
|
if (typeof row.caps !== "object" || row.caps === null || Array.isArray(row.caps)) reject("caps must be an object (the model caps snapshot at record time)");
|
|
620
|
+
const capsIssue = capsShapeIssue(row.caps);
|
|
621
|
+
if (capsIssue !== void 0) reject(capsIssue);
|
|
449
622
|
if (!Array.isArray(row.events)) reject("events must be an array");
|
|
623
|
+
const events = row.events;
|
|
624
|
+
for (const [index, event] of events.entries()) {
|
|
625
|
+
const eventIssue = eventShapeIssue(event, index);
|
|
626
|
+
if (eventIssue !== void 0) reject(eventIssue);
|
|
627
|
+
}
|
|
450
628
|
return row;
|
|
451
629
|
})
|
|
452
630
|
};
|
|
@@ -456,11 +634,17 @@ function readCassette(path) {
|
|
|
456
634
|
* hermetic CI mode; `'passthrough'` forwards unrecorded requests to the
|
|
457
635
|
* matching live adapter in `adapters` (a development convenience only).
|
|
458
636
|
*
|
|
459
|
-
* Repeated hashes replay
|
|
460
|
-
* sharing a `(adapterId, requestHash)` key form an ordered
|
|
461
|
-
* list, and every `stream()` call consumes exactly one
|
|
462
|
-
* allocated synchronously inside the call itself, so two
|
|
463
|
-
* identical requests can never be served the same
|
|
637
|
+
* Repeated hashes replay as ordered occurrences (v1.29.0 review P2):
|
|
638
|
+
* rows sharing a `(adapterId, requestHash)` key form an ordered
|
|
639
|
+
* occurrence list, and every `stream()` call consumes exactly one
|
|
640
|
+
* occurrence, allocated synchronously inside the call itself, so two
|
|
641
|
+
* concurrent identical requests can never be served the same
|
|
642
|
+
* recorded exchange. The list is sorted by the recorded `occurrence`
|
|
643
|
+
* numbers when every row of the group carries one, so concurrent
|
|
644
|
+
* identical calls whose live completions were appended out of order
|
|
645
|
+
* still replay to the callers that made them (v1.31.0 review P2); a
|
|
646
|
+
* group with any unnumbered row (recorded before v1.32.0) keeps file
|
|
647
|
+
* order.
|
|
464
648
|
* A call after the last occurrence is a miss: under `onMiss: 'throw'`
|
|
465
649
|
* it raises a VcrMissError whose `recordedOccurrences` says the hash
|
|
466
650
|
* WAS recorded but is exhausted, and under `'passthrough'` it
|
|
@@ -472,6 +656,22 @@ function readCassette(path) {
|
|
|
472
656
|
* snapshots for one `(adapterId, model)` agree, since the replay
|
|
473
657
|
* adapter can only report one caps truth per model. Violations throw
|
|
474
658
|
* a typed ConfigError naming the cassette and row.
|
|
659
|
+
*
|
|
660
|
+
* The rebuilt adapter restores the recorded provider and
|
|
661
|
+
* usageSemantics declarations (v1.30.0 review P2), so the fresh
|
|
662
|
+
* journal of a replayed run carries the same provenance stamp the
|
|
663
|
+
* recorded run got instead of silently reading like an entry from
|
|
664
|
+
* before the stamp existed. All rows of one adapter must agree on
|
|
665
|
+
* both declarations; a conflict refuses with a typed ConfigError
|
|
666
|
+
* before anything is served. A cassette recorded before v1.31.0
|
|
667
|
+
* stores no usageSemantics, so its replays stamp nothing (documented
|
|
668
|
+
* historical laxity). Under `onMiss: 'passthrough'` the recorded
|
|
669
|
+
* declarations must also match the live adapter's, absent versus
|
|
670
|
+
* present included, because a live served miss is journaled under
|
|
671
|
+
* the wrapper's declarations; a mismatch refuses at construction
|
|
672
|
+
* (v1.31.0 review P2). An adapter with no recorded rows keeps the
|
|
673
|
+
* live adapter's own declarations, so the wrapper stays a metadata
|
|
674
|
+
* preserving drop in.
|
|
475
675
|
*/
|
|
476
676
|
function replay(options) {
|
|
477
677
|
const { header, rows } = readCassette(options.cassette);
|
|
@@ -490,6 +690,7 @@ function replay(options) {
|
|
|
490
690
|
forAdapter.set(row.requestHash, occurrences);
|
|
491
691
|
byAdapter.set(row.adapterId, forAdapter);
|
|
492
692
|
}
|
|
693
|
+
for (const forAdapter of byAdapter.values()) for (const occurrences of forAdapter.values()) if (occurrences.every((row) => row.occurrence !== void 0)) occurrences.sort((a, b) => (a.occurrence ?? 0) - (b.occurrence ?? 0));
|
|
493
694
|
const live = new Map((options.adapters ?? []).map((adapter) => [adapter.id, adapter]));
|
|
494
695
|
return [.../* @__PURE__ */ new Set([...byAdapter.keys(), ...live.keys()])].map((adapterId) => {
|
|
495
696
|
const recorded = byAdapter.get(adapterId) ?? /* @__PURE__ */ new Map();
|
|
@@ -505,11 +706,23 @@ function replay(options) {
|
|
|
505
706
|
});
|
|
506
707
|
else if (existing.canonical !== canonical) throw new ConfigError(`${options.cassette} carries conflicting caps snapshots for adapter '${adapterId}' model '${row.model}'; a replay adapter reports one caps truth per model, so record the cassette again in one session`);
|
|
507
708
|
}
|
|
709
|
+
for (const field of ["provider", "usageSemantics"]) {
|
|
710
|
+
const values = [...new Set(recordedRows.map((row) => row[field]))];
|
|
711
|
+
if (values.length > 1) throw new ConfigError(`${options.cassette} carries conflicting ${field} values for adapter '${adapterId}' (${values.map((value) => value === void 0 ? "absent" : `'${value}'`).join(", ")}); a replay adapter reports one declaration per adapter, so record the cassette again in one session`);
|
|
712
|
+
}
|
|
508
713
|
const passthrough = live.get(adapterId);
|
|
714
|
+
if (someRow !== void 0 && passthrough !== void 0 && options.onMiss === "passthrough") {
|
|
715
|
+
for (const field of ["provider", "usageSemantics"]) if (someRow[field] !== passthrough[field]) {
|
|
716
|
+
const describe = (value) => value === void 0 ? "absent" : `'${value}'`;
|
|
717
|
+
throw new ConfigError(`${options.cassette} records ${field} ${describe(someRow[field])} for adapter '${adapterId}' but the live passthrough adapter declares ${describe(passthrough[field])}; the engine journals live served misses under the replay adapter declaration, so replay with a matching adapter or record the cassette again`);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
const declared = someRow ?? passthrough;
|
|
509
721
|
const cursors = /* @__PURE__ */ new Map();
|
|
510
722
|
return {
|
|
511
723
|
id: adapterId,
|
|
512
|
-
...
|
|
724
|
+
...declared?.provider === void 0 ? {} : { provider: declared.provider },
|
|
725
|
+
...declared?.usageSemantics === void 0 ? {} : { usageSemantics: declared.usageSemantics },
|
|
513
726
|
caps: (model) => {
|
|
514
727
|
const snapshot = capsByModel.get(model)?.caps ?? passthrough?.caps(model);
|
|
515
728
|
if (snapshot === void 0) throw new ConfigError(`VCR replay adapter '${adapterId}' has no caps snapshot for model '${model}'`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rulvar/testing",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.32.0",
|
|
4
4
|
"description": "Rulvar test harness: createTestEngine, FakeAdapter, VCR cassettes, replay-strict runs, matchers.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"access": "public"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@rulvar/core": "1.
|
|
29
|
+
"@rulvar/core": "1.32.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/node": "^22.20.0",
|