@copilotkit/aimock 1.37.4 → 1.38.0

package/dist/ws-realtime.js

@@ -1,5 +1,5 @@
 import { DEFAULT_TEST_ID } from "./constants.js";
-import { flattenHeaders, generateToolCallId, isContentWithToolCallsResponse, isErrorResponse, isTextResponse, isToolCallResponse, resolveFixtureBlocks, resolveResponse, resolveStrictMode, strictNoMatchLogLine, strictNoMatchMessage, strictOverrideField } from "./helpers.js";
+import { flattenHeaders, generateToolCallId, isContentWithToolCallsResponse, isErrorResponse, isTextResponse, isToolCallResponse, isTranscriptionResponse, resolveFixtureBlocks, resolveResponse, resolveStrictMode, strictNoMatchLogLine, strictNoMatchMessage, strictOverrideField } from "./helpers.js";
 import "./journal.js";
 import { matchFixtureDiagnostic } from "./router.js";
 import { calculateDelay, delay } from "./sse-writer.js";
@@ -18,6 +18,123 @@ import { randomBytes } from "node:crypto";
 function realtimeId(prefix) {
 	return `${prefix}_${randomBytes(12).toString("base64url")}`;
 }
+/**
+* Transcription `usage` is MODEL-DEPENDENT on the live API, so the admitted
+* models are split by the usage shape they actually report rather than kept in
+* one flat list. Verified against api.openai.com with a single 5s clip on BOTH
+* the realtime WS (`conversation.item.input_audio_transcription.completed`) and
+* HTTP (`POST /v1/audio/transcriptions`) surfaces:
+*
+*   whisper-1              → {"type":"duration","seconds":5}
+*   gpt-transcribe         → {"type":"duration","seconds":5}
+*   gpt-live-transcribe    → {"type":"duration","seconds":5}  (realtime only)
+*   gpt-4o-transcribe      → {"type":"tokens","total_tokens":66,…}
+*   gpt-4o-mini-transcribe → {"type":"tokens","total_tokens":66,…}
+*
+* Keeping the split here — instead of a second list beside the admission
+* predicate — means a newly admitted model has to declare its family, and the
+* two can never disagree about which models exist.
+*/
+function isDurationUsageTranscriptionModel(model) {
+	return model === "whisper-1" || model === "gpt-transcribe" || model === "gpt-live-transcribe" || model.startsWith("gpt-live-transcribe-");
+}
+function isTokenUsageTranscriptionModel(model) {
+	return model === "gpt-4o-transcribe" || model.startsWith("gpt-4o-mini-transcribe");
+}
+function isLiveTranscriptionModel(model) {
+	return isDurationUsageTranscriptionModel(model) || isTokenUsageTranscriptionModel(model);
+}
+/**
+* The synthesized `usage` for a transcription with no fixture-supplied one.
+* aimock has no real audio to measure, so the magnitudes are zero — but the
+* DISCRIMINATOR must match the model's live family, otherwise a consumer
+* branching on `usage.type` reads `undefined` counts and fails silently.
+*/
+function defaultTranscriptionUsage(model) {
+	if (isTokenUsageTranscriptionModel(model)) return {
+		type: "tokens",
+		total_tokens: 0,
+		input_tokens: 0,
+		input_token_details: {
+			text_tokens: 0,
+			audio_tokens: 0
+		},
+		output_tokens: 0
+	};
+	return {
+		type: "duration",
+		seconds: 0
+	};
+}
+function normalizeAudioFormat(value) {
+	if (value === null) return null;
+	if (typeof value === "string") return { type: value };
+	if (value && typeof value === "object" && typeof value.type === "string") return { ...value };
+	return null;
+}
+function transcriptionModel(session) {
+	return session.input_audio_transcription?.model ?? session.model;
+}
+/**
+* A live-transcription session is discriminated by the session TYPE, never by
+* the mere presence of `input_audio_transcription`.
+*
+* `input_audio_transcription` is the documented way to configure input
+* transcription on an ordinary realtime CONVERSATION session, so treating it as
+* the discriminator made the canonical realtime setup (`{model:"whisper-1"}`)
+* route `input_audio_buffer.commit` into the transcription path. That silently
+* corrupted the conversation: it appended a phantom `[audio]` turn and consumed
+* a fixture match, so the next `response.create` replayed the FOLLOWING turn's
+* content. The client saw a well-formed — but wrong — response.
+*
+* A transcription session is established explicitly, either by connecting with
+* `?intent=transcription` or via `transcription_session.update` (both set
+* `session.type`), which is what the OpenAI realtime API requires too.
+*/
+function isLiveTranscriptionSession(session) {
+	return isLiveTranscriptionModel(transcriptionModel(session)) && session.type === "transcription";
+}
+function serializeSession(session, sessionId) {
+	if (session.type === "transcription") return {
+		...sessionId ? { id: sessionId } : {},
+		object: "realtime.transcription_session",
+		type: "transcription",
+		expires_at: Math.floor(Date.now() / 1e3) + 3600,
+		audio: { input: {
+			format: session.input_audio_format,
+			noise_reduction: session.input_audio_noise_reduction,
+			transcription: session.input_audio_transcription,
+			turn_detection: session.turn_detection
+		} },
+		include: null
+	};
+	return {
+		...sessionId ? { id: sessionId } : {},
+		object: "realtime.session",
+		model: session.model,
+		expires_at: Math.floor(Date.now() / 1e3) + 3600,
+		modalities: session.modalities,
+		instructions: session.instructions,
+		tools: session.tools,
+		tool_choice: "auto",
+		temperature: session.temperature,
+		max_response_output_tokens: "inf",
+		audio: {
+			input: {
+				format: session.input_audio_format,
+				noise_reduction: session.input_audio_noise_reduction,
+				transcription: session.input_audio_transcription,
+				turn_detection: session.turn_detection
+			},
+			output: {
+				format: session.output_audio_format,
+				voice: session.voice
+			}
+		},
+		type: session.type === "conversation" || session.type === "realtime" ? "realtime" : session.type,
+		reasoning: session.reasoning
+	};
+}
 function realtimeItemsToMessages(items, instructions, logger) {
 	const messages = [];
 	if (instructions) messages.push({
@@ -145,10 +262,14 @@ function translateGAToBeta(event) {
 			const session = { ...translated.session };
 			if (session.audio && typeof session.audio === "object") {
 				const audio = session.audio;
-				session.voice = audio.voice;
-				session.input_audio_format = audio.input_audio_format;
-				session.output_audio_format = audio.output_audio_format;
-				session.input_audio_transcription = audio.input_audio_transcription;
+				const input = audio.input;
+				const output = audio.output;
+				session.voice = output?.voice;
+				session.input_audio_format = (input?.format)?.type;
+				session.output_audio_format = (output?.format)?.type;
+				session.input_audio_transcription = input?.transcription;
+				session.input_audio_noise_reduction = input?.noise_reduction;
+				session.turn_detection = input?.turn_detection ?? null;
 				delete session.audio;
 			}
 			delete session.type;
@@ -195,34 +316,13 @@ function handleWebSocketRealtime(ws, fixtures, journal, defaults) {
 		input_audio_transcription: null,
 		turn_detection: null,
 		temperature: .8,
-		type: "conversation",
+		type: defaults.transcriptionIntent ? "transcription" : "conversation",
 		reasoning: null
 	};
 	const conversationItems = [];
 	sendEvent(ws, {
 		type: "session.created",
-		session: {
-			id: sessionId,
-			object: "realtime.session",
-			model: session.model,
-			expires_at: Math.floor(Date.now() / 1e3) + 3600,
-			modalities: session.modalities,
-			instructions: session.instructions,
-			tools: session.tools,
-			tool_choice: "auto",
-			temperature: session.temperature,
-			max_response_output_tokens: "inf",
-			audio: {
-				voice: session.voice,
-				input_audio_format: session.input_audio_format,
-				output_audio_format: session.output_audio_format,
-				input_audio_noise_reduction: session.input_audio_noise_reduction,
-				input_audio_transcription: session.input_audio_transcription
-			},
-			turn_detection: session.turn_detection,
-			type: session.type,
-			reasoning: session.reasoning
-		}
+		session: serializeSession(session, sessionId)
 	}, isBeta);
 	let pending = Promise.resolve();
 	ws.on("message", (raw) => {
@@ -246,11 +346,17 @@ async function processMessage(raw, ws, fixtures, journal, defaults, session, con
 		return;
 	}
 	const msgType = parsed.type;
-	if (msgType === "session.update") {
+	const isTranscriptionSessionUpdate = msgType === "transcription_session.update";
+	if (msgType === "session.update" || isTranscriptionSessionUpdate) {
+		if (isTranscriptionSessionUpdate && !defaults.transcriptionIntent) {
+			buildErrorRealtimeEvent(ws, "transcription_session.update requires a realtime connection with intent=transcription", isBeta, "invalid_request_error", "invalid_session_config");
+			return;
+		}
 		if (parsed.session) {
 			const s = parsed.session;
 			const validTypes = new Set([
 				"conversation",
+				"realtime",
 				"transcription",
 				"translation"
 			]);
@@ -268,23 +374,35 @@ async function processMessage(raw, ws, fixtures, journal, defaults, session, con
 				}
 			}
 			const prevSession = { ...session };
+			if (isTranscriptionSessionUpdate) session.type = "transcription";
 			if (s.instructions !== void 0) session.instructions = s.instructions;
 			if (s.tools !== void 0) session.tools = s.tools;
 			if (s.modalities !== void 0) session.modalities = s.modalities;
-			if (s.model !== void 0) session.model = s.model;
 			if (s.temperature !== void 0) session.temperature = s.temperature;
 			if (s.type !== void 0) session.type = s.type;
 			if (s.audio) {
 				const audio = s.audio;
 				if (audio.voice !== void 0) session.voice = audio.voice;
-				if (audio.input_audio_format !== void 0) session.input_audio_format = audio.input_audio_format;
-				if (audio.output_audio_format !== void 0) session.output_audio_format = audio.output_audio_format;
+				if (audio.input_audio_format !== void 0) session.input_audio_format = normalizeAudioFormat(audio.input_audio_format);
+				if (audio.output_audio_format !== void 0) session.output_audio_format = normalizeAudioFormat(audio.output_audio_format);
 				if (audio.input_audio_noise_reduction !== void 0) session.input_audio_noise_reduction = audio.input_audio_noise_reduction;
 				if (audio.input_audio_transcription !== void 0) session.input_audio_transcription = audio.input_audio_transcription;
+				if (audio.input && typeof audio.input === "object") {
+					const input = audio.input;
+					if (input.transcription !== void 0) session.input_audio_transcription = input.transcription;
+					if (input.noise_reduction !== void 0) session.input_audio_noise_reduction = input.noise_reduction;
+					if (input.turn_detection !== void 0) session.turn_detection = input.turn_detection;
+					if (input.format !== void 0) session.input_audio_format = normalizeAudioFormat(input.format);
+				}
+				if (audio.output && typeof audio.output === "object") {
+					const output = audio.output;
+					if (output.voice !== void 0) session.voice = output.voice;
+					if (output.format !== void 0) session.output_audio_format = normalizeAudioFormat(output.format);
+				}
 			}
 			if (s.voice !== void 0) session.voice = s.voice;
-			if (s.input_audio_format !== void 0) session.input_audio_format = s.input_audio_format;
-			if (s.output_audio_format !== void 0) session.output_audio_format = s.output_audio_format;
+			if (s.input_audio_format !== void 0) session.input_audio_format = normalizeAudioFormat(s.input_audio_format);
+			if (s.output_audio_format !== void 0) session.output_audio_format = normalizeAudioFormat(s.output_audio_format);
 			if (s.input_audio_noise_reduction !== void 0) session.input_audio_noise_reduction = s.input_audio_noise_reduction;
 			if (s.input_audio_transcription !== void 0) session.input_audio_transcription = s.input_audio_transcription;
 			if (s.turn_detection !== void 0) session.turn_detection = s.turn_detection;
@@ -300,12 +418,13 @@ async function processMessage(raw, ws, fixtures, journal, defaults, session, con
 				"gpt-4o-mini-transcribe",
 				"gpt-realtime-translate"
 			]);
-			if (session.type === "transcription" && !transcriptionModels.has(session.model)) {
+			const candidateTranscriptionModel = transcriptionModel(session);
+			if (session.type === "transcription" && !transcriptionModels.has(candidateTranscriptionModel) && !isLiveTranscriptionSession(session)) {
 				Object.assign(session, prevSession);
 				sendEvent(ws, {
 					type: "error",
 					error: {
-						message: `Model ${s.model ?? prevSession.model} does not support session type transcription`,
+						message: `Model ${candidateTranscriptionModel} does not support session type transcription`,
 						type: "invalid_request_error",
 						code: "invalid_session_config"
 					}
@@ -326,28 +445,8 @@ async function processMessage(raw, ws, fixtures, journal, defaults, session, con
 			}
 		}
 		sendEvent(ws, {
-			type: "session.updated",
-			session: {
-				object: "realtime.session",
-				model: session.model,
-				expires_at: Math.floor(Date.now() / 1e3) + 3600,
-				modalities: session.modalities,
-				instructions: session.instructions,
-				tools: session.tools,
-				tool_choice: "auto",
-				temperature: session.temperature,
-				max_response_output_tokens: "inf",
-				audio: {
-					voice: session.voice,
-					input_audio_format: session.input_audio_format,
-					output_audio_format: session.output_audio_format,
-					input_audio_noise_reduction: session.input_audio_noise_reduction,
-					input_audio_transcription: session.input_audio_transcription
-				},
-				turn_detection: session.turn_detection,
-				type: session.type,
-				reasoning: session.reasoning
-			}
+			type: isTranscriptionSessionUpdate ? "transcription_session.updated" : "session.updated",
+			session: serializeSession(session)
 		}, isBeta);
 		return;
 	}
@@ -389,6 +488,7 @@ async function processMessage(raw, ws, fixtures, journal, defaults, session, con
 				type: "conversation.item.added",
 				item: audioItem
 			}, isBeta);
+			if (isLiveTranscriptionSession(session)) await emitLiveTranscriptionEvents(ws, fixtures, journal, defaults, session, audioItem, isBeta);
 		}
 		return;
 	}
@@ -401,6 +501,155 @@ async function processMessage(raw, ws, fixtures, journal, defaults, session, con
 		return;
 	}
 }
+async function emitLiveTranscriptionEvents(ws, fixtures, journal, defaults, session, audioItem, isBeta) {
+	const realtimeContextHeader = defaults.upgradeHeaders?.["x-aimock-context"];
+	const realtimeContext = typeof realtimeContextHeader === "string" ? realtimeContextHeader : Array.isArray(realtimeContextHeader) && realtimeContextHeader.length > 0 ? realtimeContextHeader[0] : void 0;
+	const request = {
+		model: transcriptionModel(session),
+		messages: realtimeItemsToMessages([audioItem], void 0, defaults.logger),
+		_endpointType: "realtime-transcription",
+		_context: realtimeContext
+	};
+	const testId = defaults.testId ?? DEFAULT_TEST_ID;
+	const { fixture, skippedBySequenceOrTurn } = matchFixtureDiagnostic(fixtures, request, journal.getFixtureMatchCountsForTest(testId), defaults.requestTransform);
+	const itemId = audioItem.id ?? realtimeId("item");
+	if (!fixture) {
+		if (resolveStrictMode(defaults.strict, defaults.upgradeHeaders)) {
+			const strictMessage = strictNoMatchMessage(skippedBySequenceOrTurn);
+			defaults.logger.error(strictNoMatchLogLine("WS", "/v1/realtime", skippedBySequenceOrTurn));
+			journal.add({
+				method: "WS",
+				path: "/v1/realtime",
+				headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
+				body: request,
+				response: {
+					status: 503,
+					fixture: null,
+					...strictOverrideField(defaults.strict, defaults.upgradeHeaders)
+				}
+			});
+			ws.close(1008, strictMessage);
+			return;
+		}
+		journal.add({
+			method: "WS",
+			path: "/v1/realtime",
+			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
+			body: request,
+			response: {
+				status: 404,
+				fixture: null,
+				...strictOverrideField(defaults.strict, defaults.upgradeHeaders)
+			}
+		});
+		sendLiveTranscriptionFailure(ws, itemId, {
+			message: "No fixture matched",
+			type: "invalid_request_error",
+			code: "no_fixture_match"
+		}, isBeta);
+		return;
+	}
+	const response = await resolveResponse(fixture, request);
+	if (isErrorResponse(response)) {
+		journal.incrementFixtureMatchCount(fixture, fixtures, testId);
+		journal.add({
+			method: "WS",
+			path: "/v1/realtime",
+			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
+			body: request,
+			response: {
+				status: response.status ?? 500,
+				fixture
+			}
+		});
+		sendLiveTranscriptionFailure(ws, itemId, {
+			message: response.error.message,
+			type: response.error.type ?? "server_error",
+			...response.error.code !== void 0 && { code: response.error.code }
+		}, isBeta);
+		return;
+	}
+	if (!isTranscriptionResponse(response)) {
+		journal.add({
+			method: "WS",
+			path: "/v1/realtime",
+			headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
+			body: request,
+			response: {
+				status: 500,
+				fixture
+			}
+		});
+		sendLiveTranscriptionFailure(ws, itemId, {
+			message: "Fixture response is not a transcription type",
+			type: "server_error"
+		}, isBeta);
+		return;
+	}
+	journal.incrementFixtureMatchCount(fixture, fixtures, testId);
+	const transcript = response.transcription.text;
+	const usage = response.transcription.usage ?? defaultTranscriptionUsage(request.model);
+	const journalEntry = journal.add({
+		method: "WS",
+		path: "/v1/realtime",
+		headers: flattenHeaders(defaults.upgradeHeaders ?? {}),
+		body: request,
+		response: {
+			status: 200,
+			fixture
+		}
+	});
+	const latency = fixture.latency ?? defaults.latency;
+	const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize);
+	const replaySpeed = fixture.replaySpeed ?? defaults.replaySpeed;
+	const interruption = createInterruptionSignal(fixture);
+	let chunkIndex = 0;
+	for (let index = 0; index < transcript.length; index += chunkSize) {
+		const chunkDelay = calculateDelay(chunkIndex, fixture.streamingProfile, latency, fixture.recordedTimings, replaySpeed);
+		if (chunkDelay > 0) await delay(chunkDelay, interruption?.signal);
+		if (interruption?.signal.aborted || ws.isClosed) {
+			if (interruption?.signal.aborted) {
+				journalEntry.response.interrupted = true;
+				journalEntry.response.interruptReason = interruption.reason();
+				ws.destroy();
+			}
+			interruption?.cleanup();
+			return;
+		}
+		sendEvent(ws, {
+			type: "conversation.item.input_audio_transcription.delta",
+			item_id: itemId,
+			content_index: 0,
+			delta: transcript.slice(index, index + chunkSize)
+		}, isBeta);
+		interruption?.tick();
+		chunkIndex++;
+	}
+	if (interruption?.signal.aborted) {
+		journalEntry.response.interrupted = true;
+		journalEntry.response.interruptReason = interruption.reason();
+		ws.destroy();
+		interruption.cleanup();
+		return;
+	}
+	sendEvent(ws, {
+		type: "conversation.item.input_audio_transcription.completed",
+		item_id: itemId,
+		content_index: 0,
+		transcript,
+		usage,
+		...response.transcription.languages !== void 0 ? { languages: response.transcription.languages } : {}
+	}, isBeta);
+	interruption?.cleanup();
+}
+function sendLiveTranscriptionFailure(ws, itemId, error, isBeta) {
+	sendEvent(ws, {
+		type: "conversation.item.input_audio_transcription.failed",
+		item_id: itemId,
+		content_index: 0,
+		error
+	}, isBeta);
+}
 async function handleResponseCreate(ws, fixtures, journal, defaults, session, conversationItems, isBeta, responseOverrides) {
 	const messages = realtimeItemsToMessages(conversationItems, (responseOverrides?.instructions ?? session.instructions) || void 0, defaults.logger);
 	const endpointType = {