@rulvar/testing 1.14.0 → 1.15.0

package/dist/index.d.ts

@@ -36,12 +36,25 @@ declare function replayRun<A, R>(wf: Workflow<A, R>, args: A, options: ReplayRun
 * an ordinary test run.
 */
 declare function liveTestEnabled(...requiredEnvKeys: string[]): boolean;
+/** Default total `runLiveSmoke` attempts including the first. */
+declare const DEFAULT_LIVE_SMOKE_ATTEMPTS = 3;
+/**
+* Hard ceiling on `runLiveSmoke` attempts. The helper's whole contract
+* is a bounded spend, so it refuses configurations that are not.
+*/
+declare const MAX_LIVE_SMOKE_ATTEMPTS = 10;
 interface RunLiveSmokeOptions {
-  /** Total attempts including the first (default 3, minimum 1). */
+  /**
+  * Total attempts including the first: an integer from 1 to
+  * {@link MAX_LIVE_SMOKE_ATTEMPTS} (default 3). Anything else, NaN and
+  * Infinity included, rejects with ConfigError before any stream opens.
+  */
   attempts?: number;
   /**
-  * Backoff before retry n (1-based) is `baseDelayMs * n` (default
-  * 2000). Pass 0 to retry without sleeping (unit tests).
+  * Backoff before retry n (1-based) is `baseDelayMs * n`: a
+  * non-negative integer (default 2000). Pass 0 to retry without
+  * sleeping (unit tests). Anything else rejects with ConfigError
+  * before any stream opens.
   */
   baseDelayMs?: number;
 }
@@ -67,13 +80,18 @@ type LiveSmokeOutcome = {
   status: "no-terminal";
   attempts: number;
   events: ChatEvent[];
+} | {
+  status: "contract-violation";
+  attempts: number;
+  reason: "multiple-terminals" | "terminal-not-final";
+  events: ChatEvent[];
 };
 /**
 * Drains `adapter.stream(req)` with a bounded retry policy and classifies
 * the outcome instead of throwing:
 *
-* - `'ok'`: a `finish` event arrived (the events of the successful
-*   attempt are included for further assertions).
+* - `'ok'`: the stream ended on a single terminal `finish` (the events of
+*   the successful attempt are included for further assertions).
 * - `'failed'`: a terminal error with `retryable: false`; never retried,
 *   diagnostics preserved.
 * - `'exhausted'`: every attempt ended in a `retryable: true` error; the
@@ -81,10 +99,18 @@ type LiveSmokeOutcome = {
 * - `'no-terminal'`: the stream ended with neither `finish` nor `error`,
 *   which violates the provider SPI; never retried (spending again on a
 *   misbehaving adapter is wrong).
+* - `'contract-violation'`: the stream carried more than one terminal
+*   event (`'multiple-terminals'`, e.g. an error followed by a finish) or
+*   its single terminal was not the final event
+*   (`'terminal-not-final'`). Equally an SPI violation, equally never
+*   retried, and never reported as a pass.
 *
-* Retries only ever follow typed retryable errors, so a live smoke never
-* converts a real adapter failure into a pass and never spends more than
-* `attempts` calls.
+* Retries only ever follow a well-formed stream whose single final
+* terminal is a typed retryable error, so a live smoke never converts a
+* real adapter failure or a malformed stream into a pass and never
+* spends more than `attempts` calls. Options are validated first:
+* invalid `attempts` or `baseDelayMs` reject with ConfigError before any
+* adapter call.
 */
 declare function runLiveSmoke(adapter: Pick<ProviderAdapter, "stream">, req: ChatRequest, options?: RunLiveSmokeOptions): Promise<LiveSmokeOutcome>;
 //#endregion
@@ -213,4 +239,4 @@ declare function replay(options: {
   adapters?: ProviderAdapter[];
 }): ProviderAdapter[];
 //#endregion
-export { type CassetteFixture, type CreateTestEngineOptions, FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, type FakeAdapterOptions, type FakeCall, type FakeResponder, type FakeToolCallsValue, type FakeWireErrorValue, type LiveSmokeOutcome, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, RedactFn, type ReplayRunOptions, type RunLiveSmokeOptions, type TestEngine, type TestRunHandle, VcrCassette, VcrMissError, VcrRow, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, liveTestEnabled, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash, runLiveSmoke };
+export { type CassetteFixture, type CreateTestEngineOptions, DEFAULT_LIVE_SMOKE_ATTEMPTS, FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, type FakeAdapterOptions, type FakeCall, type FakeResponder, type FakeToolCallsValue, type FakeWireErrorValue, type LiveSmokeOutcome, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, MAX_LIVE_SMOKE_ATTEMPTS, RedactFn, type ReplayRunOptions, type RunLiveSmokeOptions, type TestEngine, type TestRunHandle, VcrCassette, VcrMissError, VcrRow, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, liveTestEnabled, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash, runLiveSmoke };

package/dist/index.js

@@ -355,6 +355,27 @@ async function replayRun(wf, args, options) {
 //#endregion
 //#region src/live.ts
 /**
+* Live-test opt-in gate and the bounded live smoke.
+*
+* A provider key in the environment is not an opt-in: key-gated live
+* tests spend provider budget, so they additionally require the explicit
+* RULVAR_LIVE_TESTS=1 switch (the repository's `pnpm test:live` sets it
+* for its child run only). runLiveSmoke drains one adapter stream per
+* attempt and classifies the terminal event: a typed retryable error
+* (429 rate limit, 529 overload, transport) is retried a bounded number
+* of times with linear backoff; a non-retryable error (authentication,
+* invalid model, invalid request) fails immediately with the typed
+* WireError intact. The provider SPI requires exactly one terminal event
+* per stream, as its final event: a stream with no terminal is
+* `'no-terminal'`, one with multiple terminals or a terminal followed by
+* more events is `'contract-violation'`, and neither is ever retried
+* (spending again cannot repair a broken adapter contract). A stream
+* that THROWS propagates unchanged: adapters surface failures as typed
+* error events, so a raw throw is itself a contract violation the caller
+* must see. Options are validated before any stream is opened; invalid
+* values reject with ConfigError instead of being clamped or defaulted.
+*/
+/**
 * True only when `RULVAR_LIVE_TESTS` is exactly `'1'` AND every named
 * environment key is set to a non-empty value. Gate live tests as
 * `it.skipIf(!liveTestEnabled('ANTHROPIC_API_KEY'))(...)` so an
@@ -368,12 +389,19 @@ function liveTestEnabled(...requiredEnvKeys) {
 		return value !== void 0 && value !== "";
 	});
 }
+/** Default total `runLiveSmoke` attempts including the first. */
+const DEFAULT_LIVE_SMOKE_ATTEMPTS = 3;
+/**
+* Hard ceiling on `runLiveSmoke` attempts. The helper's whole contract
+* is a bounded spend, so it refuses configurations that are not.
+*/
+const MAX_LIVE_SMOKE_ATTEMPTS = 10;
 /**
 * Drains `adapter.stream(req)` with a bounded retry policy and classifies
 * the outcome instead of throwing:
 *
-* - `'ok'`: a `finish` event arrived (the events of the successful
-*   attempt are included for further assertions).
+* - `'ok'`: the stream ended on a single terminal `finish` (the events of
+*   the successful attempt are included for further assertions).
 * - `'failed'`: a terminal error with `retryable: false`; never retried,
 *   diagnostics preserved.
 * - `'exhausted'`: every attempt ended in a `retryable: true` error; the
@@ -381,36 +409,57 @@ function liveTestEnabled(...requiredEnvKeys) {
 * - `'no-terminal'`: the stream ended with neither `finish` nor `error`,
 *   which violates the provider SPI; never retried (spending again on a
 *   misbehaving adapter is wrong).
+* - `'contract-violation'`: the stream carried more than one terminal
+*   event (`'multiple-terminals'`, e.g. an error followed by a finish) or
+*   its single terminal was not the final event
+*   (`'terminal-not-final'`). Equally an SPI violation, equally never
+*   retried, and never reported as a pass.
 *
-* Retries only ever follow typed retryable errors, so a live smoke never
-* converts a real adapter failure into a pass and never spends more than
-* `attempts` calls.
+* Retries only ever follow a well-formed stream whose single final
+* terminal is a typed retryable error, so a live smoke never converts a
+* real adapter failure or a malformed stream into a pass and never
+* spends more than `attempts` calls. Options are validated first:
+* invalid `attempts` or `baseDelayMs` reject with ConfigError before any
+* adapter call.
 */
 async function runLiveSmoke(adapter, req, options) {
-	const attempts = Math.max(1, options?.attempts ?? 3);
-	const baseDelayMs = options?.baseDelayMs ?? 2e3;
+	const attempts = validatedAttempts(options?.attempts);
+	const baseDelayMs = validatedBaseDelayMs(options?.baseDelayMs);
 	const retryableErrors = [];
 	for (let attempt = 1; attempt <= attempts; attempt += 1) {
 		const events = [];
 		for await (const event of adapter.stream(req)) events.push(event);
-		if (events.some((event) => event.type === "finish")) return {
-			status: "ok",
+		const terminals = events.filter((event) => event.type === "finish" || event.type === "error");
+		const terminal = terminals[0];
+		if (terminal === void 0) return {
+			status: "no-terminal",
 			attempts: attempt,
 			events
 		};
-		const errorEvent = events.find((event) => event.type === "error");
-		if (errorEvent === void 0) return {
-			status: "no-terminal",
+		if (terminals.length > 1) return {
+			status: "contract-violation",
 			attempts: attempt,
+			reason: "multiple-terminals",
 			events
 		};
-		if (!errorEvent.error.retryable) return {
+		if (terminal !== events.at(-1)) return {
+			status: "contract-violation",
+			attempts: attempt,
+			reason: "terminal-not-final",
+			events
+		};
+		if (terminal.type === "finish") return {
+			status: "ok",
+			attempts: attempt,
+			events
+		};
+		if (!terminal.error.retryable) return {
 			status: "failed",
 			attempts: attempt,
-			error: errorEvent.error,
+			error: terminal.error,
 			events
 		};
-		retryableErrors.push(errorEvent.error);
+		retryableErrors.push(terminal.error);
 		if (attempt < attempts && baseDelayMs > 0) await delay(baseDelayMs * attempt);
 	}
 	return {
@@ -419,6 +468,16 @@ async function runLiveSmoke(adapter, req, options) {
 		errors: retryableErrors
 	};
 }
+function validatedAttempts(value) {
+	if (value === void 0) return 3;
+	if (!Number.isSafeInteger(value) || value < 1 || value > 10) throw new ConfigError(`runLiveSmoke attempts must be an integer from 1 to 10, got ${String(value)}`);
+	return value;
+}
+function validatedBaseDelayMs(value) {
+	if (value === void 0) return 2e3;
+	if (!Number.isSafeInteger(value) || value < 0) throw new ConfigError(`runLiveSmoke baseDelayMs must be a non-negative integer, got ${String(value)}`);
+	return value;
+}
 function delay(ms) {
 	return new Promise((resolve) => {
 		setTimeout(resolve, ms);
@@ -2089,4 +2148,4 @@ function replay(options) {
 	});
 }
 //#endregion
-export { FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, VcrMissError, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, liveTestEnabled, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash, runLiveSmoke };
+export { DEFAULT_LIVE_SMOKE_ATTEMPTS, FAKE_MODEL, FAKE_MODEL_REF, FakeAdapter, M6_ORCH_GOAL, M6_ORCH_PROFILES, M6_ORCH_RUN_ID, MAX_LIVE_SMOKE_ATTEMPTS, VcrMissError, buildFrozenV1JournalRaw, buildM2CassetteFixtures, buildV2GoldenIdentity, createTestEngine, defaultRedact, fakeToolCalls, fakeWireError, handlesInRequest, liveTestEnabled, normalizeM6Entries, readCassette, record, recordLiveCassettes, recordOrchestratorCrash, replay, replayRun, requestHash, runLiveSmoke };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulvar/testing",
-  "version": "1.14.0",
+  "version": "1.15.0",
   "description": "Rulvar test harness: createTestEngine, FakeAdapter, VCR cassettes, replay-strict runs, matchers.",
   "type": "module",
   "license": "Apache-2.0",
@@ -26,7 +26,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@rulvar/core": "1.14.0"
+    "@rulvar/core": "1.15.0"
   },
   "devDependencies": {
     "@types/node": "^22.20.0",