@copilotkit/aimock 1.44.0 → 1.45.0

package/dist/ws-realtime.d.ts

@@ -1,8 +1,8 @@
 import { Journal } from "./journal.js";
 import { Logger } from "./logger.js";
-import { ChatCompletionRequest, Fixture } from "./types.js";
+import { ChatCompletionRequest, Fixture, HandlerDefaults } from "./types.js";
 import { WebSocketConnection } from "./ws-framing.js";
-import * as node_http0 from "node:http";
+import * as node_http2 from "node:http";
 
 //#region src/ws-realtime.d.ts
 
@@ -11,12 +11,17 @@ declare function handleWebSocketRealtime(ws: WebSocketConnection, fixtures: Fixt
   chunkSize: number;
   replaySpeed?: number;
   model: string;
+  misbehavior?: HandlerDefaults["misbehavior"];
+  misbehaviorCounters?: HandlerDefaults["misbehaviorCounters"];
+  registry?: HandlerDefaults["registry"];
   logger: Logger;
   strict?: boolean;
   requestTransform?: (req: ChatCompletionRequest) => ChatCompletionRequest;
   testId?: string;
-  upgradeHeaders?: node_http0.IncomingHttpHeaders;
+  upgradeHeaders?: node_http2.IncomingHttpHeaders;
   transcriptionIntent?: boolean;
+  /** @internal Called before each message is processed (the server assigns fixture positions). */
+  beforeProcessMessage?: () => void;
 }): void;
 //#endregion
 export { handleWebSocketRealtime };

package/dist/ws-realtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ws-realtime.d.ts","names":[],"sources":["../src/ws-realtime.ts"],"sourcesContent":[],"mappings":";;;;;;;;iBAqfgB,uBAAA,KACV,+BACM,oBACD;;;;;UAMC;;2BAEiB,0BAA0B;;mBAAqB,UAAA,CAEnC"}
+{"version":3,"file":"ws-realtime.d.ts","names":[],"sources":["../src/ws-realtime.ts"],"sourcesContent":[],"mappings":";;;;;;;;iBA4hBgB,uBAAA,KACV,+BACM,oBACD;;;;;gBAMO;wBACQ;aACX;UACH;;2BAEiB,0BAA0B;;mBAAqB,UAAA,CAEnC"}

package/dist/ws-realtime.js

@@ -1,6 +1,7 @@
 import { DEFAULT_TEST_ID } from "./constants.js";
-import { flattenHeaders, generateToolCallId, isContentWithToolCallsResponse, isErrorResponse, isTextResponse, isToolCallResponse, isTranscriptionResponse, resolveFixtureBlocks, resolveResponse, resolveStrictMode, strictNoMatchLogLine, strictNoMatchMessage, strictOverrideField } from "./helpers.js";
+import { fixtureToolCallErrorCode, flattenHeaders, generateToolCallId, isCombinedFixtureResponse, isErrorResponse, isFixtureToolCallError, isTextResponse, isToolCallResponse, isTranscriptionResponse, journalFixtureToolCallError, prepareOpenAIChatMisbehavior, rejectResponsesOnlyToolCalls, requireServedFunctionToolCalls, resolveFixtureBlocks, resolveOpenAIChatMisbehaviorUsage, resolveResponse, resolveStrictMode, strictNoMatchLogLine, strictNoMatchMessage, strictOverrideField } from "./helpers.js";
 import { calculateDelay, delay } from "./sse-writer.js";
+import { planMisbehavior, recordMisbehaviorOutcome } from "./misbehavior.js";
 import { matchFixtureDiagnostic } from "./router.js";
 import "./journal.js";
 import { createInterruptionSignal } from "./interruption.js";
@@ -14,6 +15,11 @@ import { randomBytes } from "node:crypto";
 * response.create) over WebSocket and sends back Realtime API events as
 * individual WebSocket text frames.
 */
+/**
+* Wire label for custom-tool-call rejections on this transport, on both the
+* `toolCalls` and the ordered `blocks` path.
+*/
+const REALTIME_WIRE = "OpenAI Realtime";
 /** Generate a Realtime-API-style ID with underscore separator (e.g. event_xxx, item_xxx). */
 function realtimeId(prefix) {
 	return `${prefix}_${randomBytes(12).toString("base64url")}`;
@@ -135,6 +141,22 @@ function serializeSession(session, sessionId) {
 		reasoning: session.reasoning
 	};
 }
+/** Realtime declares functions directly, rather than under Chat's `function` key. */
+function realtimeToolDefinitions(tools) {
+	return tools.flatMap((value) => {
+		if (!value || typeof value !== "object") return [];
+		const tool = value;
+		if (tool.type !== "function" || typeof tool.name !== "string") return [];
+		return [{
+			type: "function",
+			function: {
+				name: tool.name,
+				...typeof tool.description === "string" ? { description: tool.description } : {},
+				...tool.parameters && typeof tool.parameters === "object" && !Array.isArray(tool.parameters) ? { parameters: tool.parameters } : {}
+			}
+		}];
+	});
+}
 function realtimeItemsToMessages(items, instructions, logger) {
 	const messages = [];
 	if (instructions) messages.push({
@@ -370,15 +392,20 @@ function handleWebSocketRealtime(ws, fixtures, journal, defaults) {
 	}, isBeta);
 	let pending = Promise.resolve();
 	ws.on("message", (raw) => {
-		pending = pending.then(() => processMessage(raw, ws, fixtures, journal, defaults, session, conversationItems, isBeta).catch((err) => {
-			const msg = err instanceof Error ? err.message : "Internal error";
-			logger.error(`WebSocket realtime error: ${msg}`);
+		pending = pending.then(async () => {
 			try {
-				buildErrorRealtimeEvent(ws, msg, isBeta, "server_error");
-			} catch (sendErr) {
-				defaults.logger.debug(`Failed to send error to client: ${sendErr instanceof Error ? sendErr.message : "unknown"}`);
+				defaults.beforeProcessMessage?.();
+				await processMessage(raw, ws, fixtures, journal, defaults, session, conversationItems, isBeta);
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : "Internal error";
+				logger.error(`WebSocket realtime error: ${msg}`);
+				try {
+					buildErrorRealtimeEvent(ws, msg, isBeta, "server_error");
+				} catch (sendErr) {
+					defaults.logger.debug(`Failed to send error to client: ${sendErr instanceof Error ? sendErr.message : "unknown"}`);
+				}
 			}
-		}));
+		});
 	});
 }
 async function processMessage(raw, ws, fixtures, journal, defaults, session, conversationItems, isBeta) {
@@ -694,6 +721,57 @@ function sendLiveTranscriptionFailure(ws, itemId, error, isBeta) {
 		error
 	}, isBeta);
 }
+/**
+* Run a fixture tool-call `check` before any output item is sent. A rejected
+* fixture (a custom tool call, which Realtime cannot carry, or a malformed
+* `toolCalls` entry or tool-call block) marks the journal entry as a 500 and
+* answers with a failed response — `response.created` then
+* `response.done` with `status: "failed"` and the error, including its aimock
+* `code`, in `status_details` — the same pair the error-fixture branch sends.
+* Returns undefined when the response was rejected.
+*/
+function checkRealtimeFixtureToolCalls(ws, journalEntry, responseId, isBeta, logger, check) {
+	try {
+		return journalFixtureToolCallError(journalEntry, check);
+	} catch (err) {
+		if (!isFixtureToolCallError(err)) throw err;
+		logger.error(`WebSocket realtime error: ${err.message}`);
+		sendEvent(ws, {
+			type: "response.created",
+			response: {
+				id: responseId,
+				object: "realtime.response",
+				status: "failed",
+				status_details: null,
+				output: [],
+				usage: null
+			}
+		}, isBeta);
+		sendEvent(ws, {
+			type: "response.done",
+			response: {
+				id: responseId,
+				object: "realtime.response",
+				status: "failed",
+				output: [],
+				status_details: {
+					type: "error",
+					error: {
+						message: err.message,
+						type: "server_error",
+						code: fixtureToolCallErrorCode(err)
+					}
+				},
+				usage: {
+					total_tokens: 0,
+					input_tokens: 0,
+					output_tokens: 0
+				}
+			}
+		}, isBeta);
+		return;
+	}
+}
 async function handleResponseCreate(ws, fixtures, journal, defaults, session, conversationItems, isBeta, responseOverrides) {
 	const messages = realtimeItemsToMessages(conversationItems, (responseOverrides?.instructions ?? session.instructions) || void 0, defaults.logger);
 	const endpointType = {
@@ -703,10 +781,12 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 	}[session.type] ?? "realtime";
 	const realtimeContextHeader = defaults.upgradeHeaders?.["x-aimock-context"];
 	const realtimeContext = typeof realtimeContextHeader === "string" ? realtimeContextHeader : Array.isArray(realtimeContextHeader) && realtimeContextHeader.length > 0 ? realtimeContextHeader[0] : void 0;
+	const tools = realtimeToolDefinitions(Array.isArray(responseOverrides?.tools) ? responseOverrides.tools : session.tools);
 	const completionReq = {
 		model: session.model,
 		messages,
 		_endpointType: endpointType,
+		...tools.length > 0 ? { tools } : {},
 		_context: realtimeContext
 	};
 	const testId = defaults.testId ?? DEFAULT_TEST_ID;
@@ -778,11 +858,24 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 		return;
 	}
 	const response = await resolveResponse(fixture, completionReq);
-	const latency = fixture.latency ?? defaults.latency;
-	const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize);
-	if (isErrorResponse(response)) {
-		const status = response.status ?? 500;
-		journal.add({
+	const plannerDefaults = {
+		...defaults,
+		replaySpeed: defaults.replaySpeed ?? 1
+	};
+	let misbehavior = planMisbehavior({
+		wire: "openai-realtime",
+		emitsToolCallIds: true,
+		fixture,
+		response,
+		request: completionReq,
+		stream: true,
+		defaults: plannerDefaults,
+		rawHeaders: defaults.upgradeHeaders ?? {},
+		url: `/v1/realtime?testId=${encodeURIComponent(testId)}`
+	});
+	if (misbehavior.kind === "applied") misbehavior = prepareOpenAIChatMisbehavior(misbehavior);
+	const addResponseEntry = (status) => {
+		const entry = journal.add({
 			method: "WS",
 			path: "/v1/realtime",
 			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
@@ -792,6 +885,47 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 				fixture
 			}
 		});
+		recordMisbehaviorOutcome({
+			entry,
+			summary: misbehavior.summary,
+			defaults: plannerDefaults,
+			testId
+		});
+		return entry;
+	};
+	if (misbehavior.kind === "error") {
+		addResponseEntry(misbehavior.status);
+		if (!misbehavior.summary?.evaluations.length) defaults.logger.error(`[misbehavior] ${misbehavior.message} (testId=${testId})`);
+		buildErrorRealtimeEvent(ws, misbehavior.message, isBeta, "invalid_request_error", misbehavior.code);
+		return;
+	}
+	const latency = fixture.latency ?? defaults.latency;
+	const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize);
+	if (misbehavior.kind === "applied") {
+		const prepared = misbehavior.response;
+		const combined = isCombinedFixtureResponse(prepared);
+		const journalEntry = addResponseEntry(200);
+		const blocks = checkRealtimeFixtureToolCalls(ws, journalEntry, responseId, isBeta, defaults.logger, () => combined && prepared.blocks?.length ? resolveFixtureBlocks(prepared.blocks) : [...(isTextResponse(prepared) || combined) && prepared.content ? [{
+			type: "text",
+			text: prepared.content
+		}] : [], ...isToolCallResponse(prepared) || combined ? (prepared.toolCalls ?? []).map((call) => ({
+			type: "toolCall",
+			...call
+		})) : []]);
+		if (!blocks) return;
+		const usage = resolveOpenAIChatMisbehaviorUsage(misbehavior, completionReq, false);
+		await streamRealtimeBlocks(ws, blocks, responseId, fixture, defaults, latency, chunkSize, isBeta, conversationItems, journalEntry, {
+			stop: misbehavior.stop,
+			usage: {
+				input_tokens: usage.prompt_tokens,
+				output_tokens: usage.completion_tokens,
+				total_tokens: usage.total_tokens
+			}
+		});
+		return;
+	}
+	if (isErrorResponse(response)) {
+		addResponseEntry(response.status ?? 500);
 		sendEvent(ws, {
 			type: "response.created",
 			response: {
@@ -827,21 +961,20 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 		}, isBeta);
 		return;
 	}
-	if (isContentWithToolCallsResponse(response)) {
-		const journalEntry = journal.add({
-			method: "WS",
-			path: "/v1/realtime",
-			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
-			body: completionReq,
-			response: {
-				status: 200,
-				fixture
-			}
-		});
+	if (isCombinedFixtureResponse(response)) {
+		const journalEntry = addResponseEntry(200);
 		if (response.blocks && response.blocks.length > 0) {
-			await streamRealtimeBlocks(ws, resolveFixtureBlocks(response.blocks), responseId, fixture, defaults, latency, chunkSize, isBeta, conversationItems, journalEntry);
+			const blocks = response.blocks;
+			const resolvedBlocks = checkRealtimeFixtureToolCalls(ws, journalEntry, responseId, isBeta, defaults.logger, () => {
+				rejectResponsesOnlyToolCalls(response, REALTIME_WIRE);
+				return resolveFixtureBlocks(blocks);
+			});
+			if (!resolvedBlocks) return;
+			await streamRealtimeBlocks(ws, resolvedBlocks, responseId, fixture, defaults, latency, chunkSize, isBeta, conversationItems, journalEntry);
 			return;
 		}
+		const functionToolCalls = checkRealtimeFixtureToolCalls(ws, journalEntry, responseId, isBeta, defaults.logger, () => requireServedFunctionToolCalls(response, REALTIME_WIRE));
+		if (!functionToolCalls) return;
 		sendEvent(ws, {
 			type: "response.created",
 			response: {
@@ -991,7 +1124,7 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 			return;
 		}
 		allOutputItems.push(textOutputItem);
-		const toolCalls = response.toolCalls ?? [];
+		const toolCalls = functionToolCalls;
 		for (let tcIdx = 0; tcIdx < toolCalls.length; tcIdx++) {
 			const tc = toolCalls[tcIdx];
 			const callId = tc.id ?? generateToolCallId();
@@ -1121,16 +1254,7 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 		return;
 	}
 	if (isTextResponse(response)) {
-		const journalEntry = journal.add({
-			method: "WS",
-			path: "/v1/realtime",
-			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
-			body: completionReq,
-			response: {
-				status: 200,
-				fixture
-			}
-		});
+		const journalEntry = addResponseEntry(200);
 		const itemId = realtimeId("item");
 		const contentIndex = 0;
 		const outputIndex = 0;
@@ -1289,16 +1413,9 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 		return;
 	}
 	if (isToolCallResponse(response)) {
-		const journalEntry = journal.add({
-			method: "WS",
-			path: "/v1/realtime",
-			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
-			body: completionReq,
-			response: {
-				status: 200,
-				fixture
-			}
-		});
+		const journalEntry = addResponseEntry(200);
+		const toolCalls = checkRealtimeFixtureToolCalls(ws, journalEntry, responseId, isBeta, defaults.logger, () => requireServedFunctionToolCalls(response, REALTIME_WIRE));
+		if (!toolCalls) return;
 		sendEvent(ws, {
 			type: "response.created",
 			response: {
@@ -1316,8 +1433,8 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 		const { recordedTimings } = fixture;
 		let interrupted = false;
 		let eventIndex = 0;
-		for (let tcIdx = 0; tcIdx < response.toolCalls.length; tcIdx++) {
-			const tc = response.toolCalls[tcIdx];
+		for (let tcIdx = 0; tcIdx < toolCalls.length; tcIdx++) {
+			const tc = toolCalls[tcIdx];
 			const callId = tc.id ?? generateToolCallId();
 			const itemId = realtimeId("item");
 			const outputItem = {
@@ -1432,16 +1549,7 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 		for (const item of outputItems) conversationItems.push(item);
 		return;
 	}
-	journal.add({
-		method: "WS",
-		path: "/v1/realtime",
-		headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
-		body: completionReq,
-		response: {
-			status: 500,
-			fixture
-		}
-	});
+	addResponseEntry(500);
 	buildErrorRealtimeEvent(ws, "Fixture response did not match any known type", isBeta, "server_error");
 }
 /**
@@ -1453,7 +1561,7 @@ async function handleResponseCreate(ws, fixtures, journal, defaults, session, co
 * as the legacy text/tool branches; a blocks-only fixture therefore never
 * produces an empty payload.
 */
-async function streamRealtimeBlocks(ws, blocks, responseId, fixture, defaults, latency, chunkSize, isBeta, conversationItems, journalEntry) {
+async function streamRealtimeBlocks(ws, blocks, responseId, fixture, defaults, latency, chunkSize, isBeta, conversationItems, journalEntry, fault) {
 	sendEvent(ws, {
 		type: "response.created",
 		response: {
@@ -1592,10 +1700,11 @@ async function streamRealtimeBlocks(ws, blocks, responseId, fixture, defaults, l
 			const callId = block.id ?? generateToolCallId();
 			const itemId = realtimeId("item");
 			const args = block.arguments;
+			const itemStatus = fault?.stop === "length" && outputIndex === blocks.length - 1 ? "incomplete" : "completed";
 			const toolOutputItem = {
 				id: itemId,
 				type: "function_call",
-				status: "completed",
+				status: itemStatus,
 				call_id: callId,
 				name: block.name,
 				arguments: args
@@ -1669,7 +1778,7 @@ async function streamRealtimeBlocks(ws, blocks, responseId, fixture, defaults, l
 					id: itemId,
 					object: "realtime.item",
 					type: "function_call",
-					status: "completed",
+					status: itemStatus,
 					call_id: callId,
 					name: block.name,
 					arguments: args
@@ -1693,9 +1802,13 @@ async function streamRealtimeBlocks(ws, blocks, responseId, fixture, defaults, l
 		response: {
 			id: responseId,
 			object: "realtime.response",
-			status: "completed",
+			status: fault?.stop === "length" || fault?.stop === "content_filter" ? "incomplete" : "completed",
+			...fault?.stop === "length" || fault?.stop === "content_filter" ? { status_details: {
+				type: "incomplete",
+				reason: fault.stop === "length" ? "max_output_tokens" : "content_filter"
+			} } : {},
 			output: allOutputItems,
-			usage: {
+			usage: fault?.usage ?? {
 				total_tokens: 0,
 				input_tokens: 0,
 				output_tokens: 0