@rulvar/testing 1.12.0 → 1.14.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { a as FAKE_MODEL, c as FakeAdapterOptions, d as FakeToolCallsValue, f as FakeWireErrorValue, i as createTestEngine, l as FakeCall, m as fakeWireError, n as TestEngine, o as FAKE_MODEL_REF, p as fakeToolCalls, r as TestRunHandle, s as FakeAdapter, t as CreateTestEngineOptions, u as FakeResponder } from "./test-engine-2Md1gvT4.js";
+import { a as FAKE_MODEL, c as FakeAdapterOptions, d as FakeToolCallsValue, f as FakeWireErrorValue, i as createTestEngine, l as FakeCall, m as fakeWireError, n as TestEngine, o as FAKE_MODEL_REF, p as fakeToolCalls, r as TestRunHandle, s as FakeAdapter, t as CreateTestEngineOptions, u as FakeResponder } from "./test-engine-DAXwdoOQ.js";
 import { AgentProfile, ChatEvent, ChatRequest, InvocationRole, JournalEntry, JournalStore, ModelCaps, ModelSpec, ProviderAdapter, ResumePreview, RunOutcome, WireError, Workflow, createEngine } from "@rulvar/core";
 
 //#region src/replay-strict.d.ts
@@ -27,6 +27,67 @@ declare function replayRun<A, R>(wf: Workflow<A, R>, args: A, options: ReplayRun
   preview: ResumePreview;
 }>;
 //#endregion
+//#region src/live.d.ts
+/**
+* True only when `RULVAR_LIVE_TESTS` is exactly `'1'` AND every named
+* environment key is set to a non-empty value. Gate live tests as
+* `it.skipIf(!liveTestEnabled('ANTHROPIC_API_KEY'))(...)` so an
+* unrelated key in the shell never triggers a paid provider call from
+* an ordinary test run.
+*/
+declare function liveTestEnabled(...requiredEnvKeys: string[]): boolean;
+interface RunLiveSmokeOptions {
+  /** Total attempts including the first (default 3, minimum 1). */
+  attempts?: number;
+  /**
+  * Backoff before retry n (1-based) is `baseDelayMs * n` (default
+  * 2000). Pass 0 to retry without sleeping (unit tests).
+  */
+  baseDelayMs?: number;
+}
+/**
+* The classified result of a bounded live smoke. `attempts` is how many
+* streams were actually opened; only `'exhausted'` reaches the
+* configured bound.
+*/
+type LiveSmokeOutcome = {
+  status: "ok";
+  attempts: number;
+  events: ChatEvent[];
+} | {
+  status: "failed";
+  attempts: number;
+  error: WireError;
+  events: ChatEvent[];
+} | {
+  status: "exhausted";
+  attempts: number;
+  errors: WireError[];
+} | {
+  status: "no-terminal";
+  attempts: number;
+  events: ChatEvent[];
+};
+/**
+* Drains `adapter.stream(req)` with a bounded retry policy and classifies
+* the outcome instead of throwing:
+*
+* - `'ok'`: a `finish` event arrived (the events of the successful
+*   attempt are included for further assertions).
+* - `'failed'`: a terminal error with `retryable: false`; never retried,
+*   diagnostics preserved.
+* - `'exhausted'`: every attempt ended in a `retryable: true` error; the
+*   per-attempt errors are preserved in order.
+* - `'no-terminal'`: the stream ended with neither `finish` nor `error`,
+*   which violates the provider SPI; never retried (spending again on a
+*   misbehaving adapter is wrong).
+*
+* Retries only ever follow typed retryable errors, so a live smoke never
+* converts a real adapter failure into a pass and never spends more than
+* `attempts` calls.
+*/
+declare function runLiveSmoke(adapter: Pick<ProviderAdapter, "stream">, req: ChatRequest, options?: RunLiveSmokeOptions): Promise<LiveSmokeOutcome>;
+//#endregion
 //#region src/cassettes/build-fixtures.d.ts
 /** One cassette fixture file: id, provenance note, and the journal. */
 /** @internal */
@@ -152,4 +213,4 @@ declare function replay(options: {
   adapters?: ProviderAdapter[];
 }): ProviderAdapter[];
 //#endregion
-export { type CassetteFixture, type CreateTestEngineOptions, FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, type FakeAdapterOptions, type FakeCall, type FakeResponder, type FakeToolCallsValue, type FakeWireErrorValue, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, RedactFn, type ReplayRunOptions, type TestEngine, type TestRunHandle, VcrCassette, VcrMissError, VcrRow, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash };
+export { type CassetteFixture, type CreateTestEngineOptions, FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, type FakeAdapterOptions, type FakeCall, type FakeResponder, type FakeToolCallsValue, type FakeWireErrorValue, type LiveSmokeOutcome, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, RedactFn, type ReplayRunOptions, type RunLiveSmokeOptions, type TestEngine, type TestRunHandle, VcrCassette, VcrMissError, VcrRow, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, liveTestEnabled, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash, runLiveSmoke };

package/dist/index.js

@@ -7,7 +7,10 @@ import { createHash } from "node:crypto";
 * FakeAdapter (M1-T14): a REAL ProviderAdapter that resolves calls from
 * declared patterns instead of the network, behind the same seam as live
 * adapters, so unit tests run through the full engine: journal, scheduler,
-* budget layers, and event stream. Calls cost zero USD.
+* budget layers, and event stream. Calls cost zero USD. Honors the
+* caller's AbortSignal exactly like a live adapter: an abort ends the
+* stream promptly with no terminal event, so cancellation, deadline, and
+* budget tests observe the same journal shapes as production adapters.
 */
 /** Scripts a tool-calling turn from a responder. */
 function fakeToolCalls(...calls) {
@@ -49,6 +52,38 @@ const FAKE_CAPS = {
 		outputUsdPerMTok: 0
 	}
 };
+/**
+* Races a responder promise against the caller's abort. On abort the
+* pending responder is detached: its eventual value is discarded and an
+* eventual rejection is swallowed (the caller cancelled; a late failure
+* of the abandoned work must not become an unhandled rejection). A
+* rejection that settles first propagates to the caller unchanged.
+*/
+async function raceAbort(pending, signal) {
+	if (signal === void 0) return {
+		aborted: false,
+		value: await pending
+	};
+	if (signal.aborted) {
+		pending.catch(() => void 0);
+		return { aborted: true };
+	}
+	let onAbort = () => void 0;
+	const abortPromise = new Promise((resolve) => {
+		onAbort = () => resolve({ aborted: true });
+		signal.addEventListener("abort", onAbort, { once: true });
+	});
+	try {
+		const raced = await Promise.race([pending.then((value) => ({
+			aborted: false,
+			value
+		})), abortPromise]);
+		if (raced.aborted) pending.catch(() => void 0);
+		return raced;
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+	}
+}
 function lastUserText(req) {
 	for (let i = req.messages.length - 1; i >= 0; i -= 1) {
 		const msg = req.messages[i];
@@ -61,7 +96,10 @@ var FakeAdapter = class {
 	id = "fake";
 	agents;
 	mintId = createCanonicalIdMinter();
-	/** Every request this adapter served, in order. */
+	/**
+	* Every request this adapter served, in order. A request whose signal
+	* was already aborted on arrival was never served and is not recorded.
+	*/
 	calls = [];
 	constructor(options) {
 		this.agents = options.agents;
@@ -83,7 +121,9 @@ var FakeAdapter = class {
 		}
 		return fallback;
 	}
-	async *stream(req) {
+	async *stream(req, signal) {
+		const aborted = () => signal?.aborted === true;
+		if (aborted()) return;
 		const telemetry = req.providerOptions?.rulvar ?? {};
 		const call = {
 			prompt: lastUserText(req),
@@ -107,8 +147,13 @@ var FakeAdapter = class {
 		}
 		let value;
 		try {
-			value = typeof responder === "function" ? await responder(call) : responder;
+			if (typeof responder === "function") {
+				const raced = await raceAbort(Promise.resolve(responder(call)), signal);
+				if (raced.aborted) return;
+				value = raced.value;
+			} else value = responder;
 		} catch (thrown) {
+			if (aborted()) return;
 			yield {
 				type: "error",
 				error: {
@@ -120,14 +165,12 @@ var FakeAdapter = class {
 			};
 			return;
 		}
-		if (isFakeWireError(value)) {
-			yield {
-				type: "error",
-				error: value.error
-			};
-			return;
-		}
-		if (isFakeToolCalls(value)) {
+		const events = [];
+		if (isFakeWireError(value)) events.push({
+			type: "error",
+			error: value.error
+		});
+		else if (isFakeToolCalls(value)) {
 			const usage = {
 				inputTokens: Math.max(1, Math.ceil(call.prompt.length / 4)),
 				outputTokens: Math.max(1, value.calls.length * 8),
@@ -136,66 +179,70 @@ var FakeAdapter = class {
 			};
 			for (const toolCall of value.calls) {
 				const id = this.mintId();
-				yield {
+				events.push({
 					type: "tool-call-start",
 					id,
 					name: toolCall.name
-				};
-				yield {
+				});
+				events.push({
 					type: "tool-call-end",
 					id,
 					args: toolCall.args
-				};
+				});
 			}
-			yield {
+			events.push({
 				type: "finish",
 				finish: { reason: "tool-calls" },
 				usage
+			});
+		} else {
+			const text = typeof value === "string" ? value : JSON.stringify(value);
+			const usage = {
+				inputTokens: Math.max(1, Math.ceil(call.prompt.length / 4)),
+				outputTokens: Math.max(1, Math.ceil(text.length / 4)),
+				cacheReadTokens: 0,
+				cacheWriteTokens: 0
 			};
-			return;
-		}
-		const text = typeof value === "string" ? value : JSON.stringify(value);
-		const usage = {
-			inputTokens: Math.max(1, Math.ceil(call.prompt.length / 4)),
-			outputTokens: Math.max(1, Math.ceil(text.length / 4)),
-			cacheReadTokens: 0,
-			cacheWriteTokens: 0
-		};
-		const forcedName = typeof req.toolChoice === "object" ? req.toolChoice.name : void 0;
-		if (forcedName !== void 0) {
-			let args = value;
-			if (typeof value === "string") try {
-				args = JSON.parse(value);
-			} catch {
-				args = { text: value };
+			const forcedName = typeof req.toolChoice === "object" ? req.toolChoice.name : void 0;
+			if (forcedName !== void 0) {
+				let args = value;
+				if (typeof value === "string") try {
+					args = JSON.parse(value);
+				} catch {
+					args = { text: value };
+				}
+				const id = this.mintId();
+				events.push({
+					type: "tool-call-start",
+					id,
+					name: forcedName
+				});
+				events.push({
+					type: "tool-call-end",
+					id,
+					args
+				});
+				events.push({
+					type: "finish",
+					finish: { reason: "tool-calls" },
+					usage
+				});
+			} else {
+				events.push({
+					type: "text-delta",
+					text
+				});
+				events.push({
+					type: "finish",
+					finish: { reason: "stop" },
+					usage
+				});
 			}
-			const id = this.mintId();
-			yield {
-				type: "tool-call-start",
-				id,
-				name: forcedName
-			};
-			yield {
-				type: "tool-call-end",
-				id,
-				args
-			};
-			yield {
-				type: "finish",
-				finish: { reason: "tool-calls" },
-				usage
-			};
-			return;
 		}
-		yield {
-			type: "text-delta",
-			text
-		};
-		yield {
-			type: "finish",
-			finish: { reason: "stop" },
-			usage
-		};
+		for (const event of events) {
+			if (aborted()) return;
+			yield event;
+		}
 	}
 };
 //#endregion
@@ -306,6 +353,78 @@ async function replayRun(wf, args, options) {
 	};
 }
 //#endregion
+//#region src/live.ts
+/**
+* True only when `RULVAR_LIVE_TESTS` is exactly `'1'` AND every named
+* environment key is set to a non-empty value. Gate live tests as
+* `it.skipIf(!liveTestEnabled('ANTHROPIC_API_KEY'))(...)` so an
+* unrelated key in the shell never triggers a paid provider call from
+* an ordinary test run.
+*/
+function liveTestEnabled(...requiredEnvKeys) {
+	if (process.env.RULVAR_LIVE_TESTS !== "1") return false;
+	return requiredEnvKeys.every((key) => {
+		const value = process.env[key];
+		return value !== void 0 && value !== "";
+	});
+}
+/**
+* Drains `adapter.stream(req)` with a bounded retry policy and classifies
+* the outcome instead of throwing:
+*
+* - `'ok'`: a `finish` event arrived (the events of the successful
+*   attempt are included for further assertions).
+* - `'failed'`: a terminal error with `retryable: false`; never retried,
+*   diagnostics preserved.
+* - `'exhausted'`: every attempt ended in a `retryable: true` error; the
+*   per-attempt errors are preserved in order.
+* - `'no-terminal'`: the stream ended with neither `finish` nor `error`,
+*   which violates the provider SPI; never retried (spending again on a
+*   misbehaving adapter is wrong).
+*
+* Retries only ever follow typed retryable errors, so a live smoke never
+* converts a real adapter failure into a pass and never spends more than
+* `attempts` calls.
+*/
+async function runLiveSmoke(adapter, req, options) {
+	const attempts = Math.max(1, options?.attempts ?? 3);
+	const baseDelayMs = options?.baseDelayMs ?? 2e3;
+	const retryableErrors = [];
+	for (let attempt = 1; attempt <= attempts; attempt += 1) {
+		const events = [];
+		for await (const event of adapter.stream(req)) events.push(event);
+		if (events.some((event) => event.type === "finish")) return {
+			status: "ok",
+			attempts: attempt,
+			events
+		};
+		const errorEvent = events.find((event) => event.type === "error");
+		if (errorEvent === void 0) return {
+			status: "no-terminal",
+			attempts: attempt,
+			events
+		};
+		if (!errorEvent.error.retryable) return {
+			status: "failed",
+			attempts: attempt,
+			error: errorEvent.error,
+			events
+		};
+		retryableErrors.push(errorEvent.error);
+		if (attempt < attempts && baseDelayMs > 0) await delay(baseDelayMs * attempt);
+	}
+	return {
+		status: "exhausted",
+		attempts,
+		errors: retryableErrors
+	};
+}
+function delay(ms) {
+	return new Promise((resolve) => {
+		setTimeout(resolve, ms);
+	});
+}
+//#endregion
 //#region src/cassettes/build-fixtures.ts
 /**
 * M2 cassette and frozen-fixture builders (M2-T12). Fixtures are
@@ -1970,4 +2089,4 @@ function replay(options) {
 	});
 }
 //#endregion
-export { FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, VcrMissError, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash };
+export { FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, VcrMissError, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, liveTestEnabled, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash, runLiveSmoke };

package/dist/matchers.d.ts

@@ -1,4 +1,4 @@
-import { r as TestRunHandle } from "./test-engine-2Md1gvT4.js";
+import { r as TestRunHandle } from "./test-engine-DAXwdoOQ.js";
 
 //#region src/matchers.d.ts
 interface MatcherResult {

package/dist/{test-engine-2Md1gvT4.d.ts → test-engine-DAXwdoOQ.d.ts}

@@ -48,12 +48,15 @@ declare class FakeAdapter implements ProviderAdapter {
   readonly id = "fake";
   private readonly agents;
   private readonly mintId;
-  /** Every request this adapter served, in order. */
+  /**
+  * Every request this adapter served, in order. A request whose signal
+  * was already aborted on arrival was never served and is not recorded.
+  */
   readonly calls: FakeCall[];
   constructor(options: FakeAdapterOptions);
   caps(this: void): ModelCaps;
   private match;
-  stream(req: ChatRequest): AsyncIterable<ChatEvent>;
+  stream(req: ChatRequest, signal?: AbortSignal): AsyncIterable<ChatEvent>;
 }
 //#endregion
 //#region src/test-engine.d.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/testing",
-  "version": "1.12.0",
+  "version": "1.14.0",
   "description": "Rulvar test harness: createTestEngine, FakeAdapter, VCR cassettes, replay-strict runs, matchers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -26,7 +26,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.12.0"
+    "@rulvar/core": "1.14.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",