@copilotkit/aimock 1.45.1 → 1.47.0

package/dist/fixture-loader.js

@@ -1,8 +1,10 @@
 import { MCP_FAKES_ECHO_LIMIT } from "./constants.js";
 import { build, fixed, jsonText, msg, quote, within } from "./message-text.js";
-import { fixtureListTarget, isAudioResponse, isCombinedFixtureResponse, isEmbeddingResponse, isErrorResponse, isImageResponse, isJSONResponse, isLiveResponse, isTextResponse, isToolCallResponse, isTranscriptionResponse, isVideoResponse } from "./helpers.js";
+import { fixtureCitationsProblem, fixtureFilesProblem, fixtureListTarget, isAudioResponse, isCombinedFixtureResponse, isEmbeddingResponse, isErrorResponse, isImageResponse, isJSONResponse, isLiveResponse, isRawResponse, isTextResponse, isToolCallResponse, isTranscriptionResponse, isVideoResponse, resolveServedBlockOutcome } from "./helpers.js";
 import { normalizeLiveFixture } from "./live-fixture.js";
+import { checkRawResponse } from "./raw-validation.js";
 import { CHAOS_FIELDS, CHAOS_FIELD_NAMES, parseChaosField } from "./chaos.js";
+import { getUsageFieldDefinition } from "./response/usage/fields.js";
 import { copyFixtureMisbehaviorPosition, parseMisbehavior, setFixtureMisbehaviorPosition, validateFixtureMisbehavior } from "./misbehavior.js";
 import { readFileSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
@@ -69,6 +71,7 @@ function entryToFixture(entry, logger, liveOptions, source) {
 			userMessage: entry.match.userMessage,
 			systemMessage: entry.match.systemMessage,
 			inputText: entry.match.inputText,
+			...entry.match.embeddingRequest !== void 0 && { embeddingRequest: entry.match.embeddingRequest },
 			toolCallId: entry.match.toolCallId,
 			toolResultContains: entry.match.toolResultContains,
 			toolName: entry.match.toolName,
@@ -90,6 +93,8 @@ function entryToFixture(entry, logger, liveOptions, source) {
 		...entry.replaySpeed != null && { replaySpeed: entry.replaySpeed },
 		...entry.chaos !== void 0 && { chaos: entry.chaos },
 		...entry.openRouterProcessing !== void 0 && { openRouterProcessing: entry.openRouterProcessing },
+		...entry.nonStreamingTiming !== void 0 && { nonStreamingTiming: entry.nonStreamingTiming },
+		...entry.nonStreamingTimingAllProviders !== void 0 && { nonStreamingTimingAllProviders: entry.nonStreamingTimingAllProviders },
 		...entry.metadata !== void 0 && { metadata: entry.metadata }
 	};
 	if (source) setFixtureMisbehaviorPosition(fixture, `${source.file}#${source.index}`);
@@ -297,6 +302,52 @@ function validateReasoning(response, fixtureIndex, results) {
 		});
 	}
 }
+/** The wires that read `files` and `citations`; every other wire ignores them. */
+const CITATIONS_FILES_WIRES = "Chat Completions, Anthropic Messages, Gemini or the OpenAI Responses API";
+/**
+* Generated `files` and source `citations`. A malformed value is not a load
+* error (registration never throws for it, as in the release that added the
+* keys); each request the fixture serves on Chat Completions, Anthropic,
+* Gemini or the Responses API answers a coded 500 instead, so warn here with
+* the same message. Citations are checked against the text they cite when
+* served ({@link citedText}). A turn with no text (tool calls only, blocks
+* with no text block, or empty content) has nothing to cite, so any
+* non-empty `citations` there is reported exactly as the wires report it.
+*/
+function validateCitationsAndFiles(response, fixtureIndex, results) {
+	const filesProblem = fixtureFilesProblem(response.files);
+	if (filesProblem !== void 0) results.push({
+		severity: "warning",
+		fixtureIndex,
+		message: `${filesProblem} — requests it serves on ${CITATIONS_FILES_WIRES} answer a 500 (aimock_invalid_fixture_file)`
+	});
+	const text = citedText(response);
+	if (text === void 0) return;
+	const citationsProblem = fixtureCitationsProblem(response.citations, text);
+	if (citationsProblem !== void 0) results.push({
+		severity: "warning",
+		fixtureIndex,
+		message: `${citationsProblem} — requests it serves on ${CITATIONS_FILES_WIRES} answer a 500 (aimock_invalid_fixture_citation)`
+	});
+}
+/**
+* The text a fixture's citations index when served: the joined text of every
+* text block when `responsesBlocks` or `blocks` is non-empty (through the
+* same {@link resolveServedBlockOutcome} the providers use; empty when no
+* block is text), else a string `content`, else empty for a tool-only turn.
+* Empty text means a turn with nothing to cite. Undefined for a response
+* that is neither (validated elsewhere) and for a malformed blocks array,
+* which `validateBlocks` reports as an error.
+*/
+function citedText(response) {
+	if (Array.isArray(response.blocks) && response.blocks.length > 0 || Array.isArray(response.responsesBlocks) && response.responsesBlocks.length > 0) try {
+		return resolveServedBlockOutcome(response)?.content;
+	} catch {
+		return;
+	}
+	if (typeof response.content === "string") return response.content;
+	if (Array.isArray(response.toolCalls) || Array.isArray(response.customToolCalls)) return "";
+}
 function validateWebSearches(response, fixtureIndex, results) {
 	if (response.webSearches !== void 0) if (!Array.isArray(response.webSearches)) results.push({
 		severity: "error",
@@ -679,11 +730,29 @@ function validateFixtures(fixtures, liveOptions) {
 			});
 		}
 		if (typeof response === "function") {} else {
-			if (!isCombinedFixtureResponse(response) && !isTextResponse(response) && !isToolCallResponse(response) && !isErrorResponse(response) && !isEmbeddingResponse(response) && !isImageResponse(response) && !isAudioResponse(response) && !isTranscriptionResponse(response) && !isVideoResponse(response) && !isJSONResponse(response) && !isLiveResponse(response)) results.push({
+			if (!isCombinedFixtureResponse(response) && !isTextResponse(response) && !isToolCallResponse(response) && !isErrorResponse(response) && !isEmbeddingResponse(response) && !isImageResponse(response) && !isAudioResponse(response) && !isTranscriptionResponse(response) && !isVideoResponse(response) && !isJSONResponse(response) && !isLiveResponse(response) && !isRawResponse(response)) results.push({
 				severity: "error",
 				fixtureIndex: i,
-				message: "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, or live)"
+				message: "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, json, live, or raw)"
 			});
+			if (isRawResponse(response)) {
+				const context = rawFixtureContext(original, response);
+				for (const finding of checkRawResponse(response.raw, context)) results.push({
+					severity: finding.severity,
+					fixtureIndex: i,
+					message: finding.message
+				});
+				if (f.chunkSize !== void 0) results.push({
+					severity: "warning",
+					fixtureIndex: i,
+					message: "chunkSize is ignored for raw fixtures: each event is one frame"
+				});
+				if (f.openRouterProcessing !== void 0) results.push({
+					severity: "warning",
+					fixtureIndex: i,
+					message: "openRouterProcessing is ignored for raw fixtures"
+				});
+			}
 			const hasNonEmptyBlocks = Array.isArray(response.blocks) && response.blocks.length > 0 || Array.isArray(response.responsesBlocks) && response.responsesBlocks.length > 0;
 			const hasCustomToolCalls = Array.isArray(response.customToolCalls) && response.customToolCalls.length > 0;
 			if (isTextResponse(response)) {
@@ -732,6 +801,7 @@ function validateFixtures(fixtures, liveOptions) {
 			}
 			validateBlocks(response, i, results);
 			validateResponsesToolKeys(response, i, results);
+			validateCitationsAndFiles(response, i, results);
 			if (isToolCallResponse(response)) {
 				if (response.toolCalls.length === 0 && !hasCustomToolCalls) results.push({
 					severity: "warning",
@@ -829,6 +899,11 @@ function validateFixtures(fixtures, liveOptions) {
 					fixtureIndex: i,
 					message: `override "role" must be a string, got ${typeof r.role}`
 				});
+				if (r.phase !== void 0 && r.phase !== "commentary" && r.phase !== "final_answer") results.push({
+					severity: "error",
+					fixtureIndex: i,
+					message: "override \"phase\" must be \"commentary\" or \"final_answer\""
+				});
 				if (r.systemFingerprint !== void 0 && typeof r.systemFingerprint !== "string") results.push({
 					severity: "error",
 					fixtureIndex: i,
@@ -839,52 +914,38 @@ function validateFixtures(fixtures, liveOptions) {
 					fixtureIndex: i,
 					message: `override "usage" must be an object`
 				});
-				else {
-					const numericInnerFields = {
-						cost_details: [
-							"upstream_inference_cost",
-							"upstream_inference_prompt_cost",
-							"upstream_inference_completions_cost"
-						],
-						prompt_tokens_details: [
-							"cached_tokens",
-							"cache_write_tokens",
-							"audio_tokens"
-						],
-						completion_tokens_details: ["reasoning_tokens"]
-					};
-					const objectUsageFields = new Set(Object.keys(numericInnerFields));
-					for (const key of Object.keys(r.usage)) {
-						const val = r.usage[key];
-						if (val === void 0) continue;
-						if (objectUsageFields.has(key)) if (typeof val !== "object" || val === null || Array.isArray(val)) results.push({
-							severity: "error",
-							fixtureIndex: i,
-							message: `override "usage.${key}" must be an object, got ${Array.isArray(val) ? "array" : typeof val}`
-						});
-						else {
-							const inner = val;
-							for (const innerKey of numericInnerFields[key]) {
-								const innerVal = inner[innerKey];
-								if (innerVal !== void 0 && typeof innerVal !== "number") results.push({
-									severity: "error",
-									fixtureIndex: i,
-									message: `override "usage.${key}.${innerKey}" must be a number, got ${typeof innerVal}`
-								});
-							}
-						}
-						else if (key === "is_byok") {
-							if (typeof val !== "boolean") results.push({
+				else for (const key of Object.keys(r.usage)) {
+					const val = r.usage[key];
+					if (val === void 0) continue;
+					const definition = getUsageFieldDefinition(key);
+					if (definition.kind === "number-or-object" && typeof val === "number") continue;
+					if (definition.kind === "object" || definition.kind === "number-or-object") if (typeof val !== "object" || val === null || Array.isArray(val)) results.push({
+						severity: "error",
+						fixtureIndex: i,
+						message: `override "usage.${key}" must be ${definition.kind === "number-or-object" ? "a number or object" : "an object"}, got ${Array.isArray(val) ? "array" : typeof val}`
+					});
+					else {
+						const inner = val;
+						for (const innerKey of definition.numericMembers) {
+							const innerVal = inner[innerKey];
+							if (innerVal !== void 0 && typeof innerVal !== "number") results.push({
 								severity: "error",
 								fixtureIndex: i,
-								message: `override "usage.is_byok" must be a boolean, got ${typeof val}`
+								message: `override "usage.${key}.${innerKey}" must be a number, got ${typeof innerVal}`
 							});
-						} else if (typeof val !== "number") results.push({
+						}
+					}
+					else if (definition.kind === "boolean") {
+						if (typeof val !== "boolean") results.push({
 							severity: "error",
 							fixtureIndex: i,
-							message: `override "usage.${key}" must be a number, got ${typeof val}`
+							message: `override "usage.is_byok" must be a boolean, got ${typeof val}`
 						});
-					}
+					} else if (typeof val !== "number") results.push({
+						severity: "error",
+						fixtureIndex: i,
+						message: `override "usage.${key}" must be a number, got ${typeof val}`
+					});
 				}
 			}
 		}
@@ -908,6 +969,14 @@ function validateFixtures(fixtures, liveOptions) {
 			fixtureIndex: i,
 			message: "disconnectAfterMs must be >= 0"
 		});
+		for (const flag of ["nonStreamingTiming", "nonStreamingTimingAllProviders"]) {
+			const value = f[flag];
+			if (value !== void 0 && typeof value !== "boolean") results.push({
+				severity: "warning",
+				fixtureIndex: i,
+				message: `${flag} must be a boolean, got ${typeof value} — the opt-in is ignored`
+			});
+		}
 		if (f.streamingProfile !== void 0) {
 			const sp = f.streamingProfile;
 			if (sp.ttft !== void 0 && sp.ttft < 0) results.push({
@@ -1036,7 +1105,8 @@ function validateFixtures(fixtures, liveOptions) {
 				m.sequenceIndex,
 				m.turnIndex,
 				m.hasToolResult,
-				...m.toolNamespace !== void 0 ? [`toolNamespace=${m.toolNamespace}`] : []
+				...m.toolNamespace !== void 0 ? [`toolNamespace=${m.toolNamespace}`] : [],
+				...m.embeddingRequest !== void 0 ? [JSON.stringify(m.embeddingRequest)] : []
 			].map((v) => v === void 0 ? "" : String(v)).join("|");
 			const prev = seenUserMessages.get(dedupKey);
 			if (prev !== void 0) {
@@ -1054,7 +1124,7 @@ function validateFixtures(fixtures, liveOptions) {
 			} else seenUserMessages.set(dedupKey, i);
 		}
 		const match = f.match;
-		if (!(match.endpoint !== void 0 || match.context !== void 0 || match.userMessage !== void 0 || match.systemMessage !== void 0 || match.inputText !== void 0 || match.responseFormat !== void 0 || match.toolCallId !== void 0 || match.toolResultContains !== void 0 || match.toolName !== void 0 || match.toolNamespace !== void 0 || match.model !== void 0 || match.predicate !== void 0 || match.turnIndex !== void 0 || match.sequenceIndex !== void 0 || match.hasToolResult !== void 0) && i < fixtures.length - 1) {
+		if (!(match.endpoint !== void 0 || match.context !== void 0 || match.userMessage !== void 0 || match.systemMessage !== void 0 || match.inputText !== void 0 || match.embeddingRequest !== void 0 || match.responseFormat !== void 0 || match.toolCallId !== void 0 || match.toolResultContains !== void 0 || match.toolName !== void 0 || match.toolNamespace !== void 0 || match.model !== void 0 || match.predicate !== void 0 || match.turnIndex !== void 0 || match.sequenceIndex !== void 0 || match.hasToolResult !== void 0) && i < fixtures.length - 1) {
 			const ref = {
 				rule: "catch-all-not-last",
 				shadowsFrom: i + 1,
@@ -1298,6 +1368,11 @@ const misbehaviorEnabledFixtures = /* @__PURE__ */ new WeakSet();
 function enableHeldFixtureMisbehavior(fixture) {
 	const held = heldFixtureMisbehavior.get(fixture);
 	if (held === void 0) return;
+	if (typeof fixture.response !== "function" && isRawResponse(fixture.response)) throw new MisbehaviorConfigError({
+		rule: "misbehavior/not-applicable",
+		file: held.file,
+		detail: `${held.path}: misbehavior does not apply to raw fixtures`
+	});
 	const parsed = parseMisbehavior(held.value, held.path);
 	if (parsed.ok) fixture.misbehavior = parsed.config;
 	const issue = parsed.ok ? validateFixtureMisbehavior(fixture, held.path) : parsed.issue;
@@ -1310,6 +1385,26 @@ function enableHeldFixtureMisbehavior(fixture) {
 	misbehaviorEnabledFixtures.add(fixture);
 }
 /**
+* @internal The facts about a raw fixture that {@link checkRawResponse} checks
+* beyond `raw` itself, for `served`, the response as this server reads it.
+* Load validation and serve time both use it, so they agree:
+*
+* - `otherResponseKeys`: top-level keys of the fixture's response as written
+*   that `served` lacks. Only `customToolCalls` / `responsesBlocks` are ever
+*   dropped (on a legacy server), and a fixture that writes either next to
+*   `raw` is not a raw fixture to the router.
+* - `fixtureMisbehavior`: the fixture sets its own `misbehavior`, recognized
+*   or still held.
+*/
+function rawFixtureContext(fixture, served) {
+	const source = responsesToolsViewSource(fixture);
+	const written = fixture.response;
+	return {
+		otherResponseKeys: typeof written === "function" ? [] : Object.keys(written).filter((key) => !(key in served)),
+		fixtureMisbehavior: fixture.misbehavior !== void 0 || heldFixtureMisbehavior.has(source)
+	};
+}
+/**
 * @internal Mark a fixture whose `misbehavior` a server with misbehavior
 * enabled has recognized, so `validateFixtures` checks it.
 */
@@ -1318,5 +1413,5 @@ function markFixtureMisbehaviorEnabled(fixture) {
 }
 
 //#endregion
-export { FixtureLoadError, INJECTED_STATUS_RANGE, MisbehaviorConfigError, claimOneShotError, clearFixtureQueue, enableHeldFixtureMisbehavior, entryToFixture, fixturesOf, hasMcpFakesKey, isInjectableStatus, isOneShotError, listFixtureDir, loadFixtureFile, loadFixturesFromDir, markFixtureMisbehaviorEnabled, markFixtureResponsesToolsExtended, normalizeResponse, normalizeResponsesToolsKeys, queueOneShotError, readFixtureJson, releaseOneShotError, renderValidationRef, responsesToolsExtendedView, responsesToolsViewSource, validateFixtures };
+export { FixtureLoadError, INJECTED_STATUS_RANGE, MisbehaviorConfigError, claimOneShotError, clearFixtureQueue, enableHeldFixtureMisbehavior, entryToFixture, fixturesOf, hasMcpFakesKey, isInjectableStatus, isOneShotError, listFixtureDir, loadFixtureFile, loadFixturesFromDir, markFixtureMisbehaviorEnabled, markFixtureResponsesToolsExtended, normalizeResponse, normalizeResponsesToolsKeys, queueOneShotError, rawFixtureContext, readFixtureJson, releaseOneShotError, renderValidationRef, responsesToolsExtendedView, responsesToolsViewSource, validateFixtures };
 //# sourceMappingURL=fixture-loader.js.map