@xopcai/xopc 0.0.241 → 0.0.243

package/dist/src/voice/realtime/runtime.js

@@ -0,0 +1,622 @@
+import { createLogger } from "../../utils/logger/index.js";
+import { init_logger } from "../../utils/logger.js";
+import { onUserContextChange } from "../../user-context/changes.js";
+import { getModelCatalogStore, init_model_catalog_store } from "../../providers/model-catalog-store.js";
+import { ALIBABA_REALTIME_TTS_MODEL } from "../tts/providers/alibaba-speech.js";
+import { resolveSpeechProviderChain } from "../tts/factory.js";
+import { mergeTtsConfigFromAppConfig } from "../tts/merge-config.js";
+import { getMediaUnderstandingProvider } from "../../media-understanding/registry.js";
+import { resolveSTTProviderChain } from "../stt/factory.js";
+import { mergeSttConfigFromAppConfig } from "../stt/config.js";
+import { voiceConversationInstructions } from "./conversation-context.js";
+import { createPreauthConnectionBudget } from "../../gateway/security/preauth-connection-budget.js";
+import { encodeVoiceAudioFrame } from "../../../packages/realtime-protocol/src/voice-audio.js";
+import { VOICE_REALTIME_HEARTBEAT_INTERVAL_MS, VOICE_REALTIME_MAX_BINARY_FRAME_BYTES, VOICE_REALTIME_START_TIMEOUT_MS, VOICE_REALTIME_WS_PATH, parseVoiceClientJsonFrame, parseVoiceClientMessage } from "../../../packages/realtime-protocol/src/voice.js";
+import { createAgentVoiceEngine } from "./agentEngine.js";
+import { resolveOmniRoute } from "./omniRoute.js";
+import { createOmniVoiceEngine } from "./omniEngine.js";
+import { createRequire } from "node:module";
+import crypto from "node:crypto";
+//#region src/voice/realtime/runtime.ts
+init_logger();
+init_model_catalog_store();
+const { WebSocket: WebSocketState, WebSocketServer } = createRequire(import.meta.url)("ws");
+const log = createLogger("Voice:Realtime");
+const TICKET_TTL_MS = 6e4;
+const MAX_OUTSTANDING_TICKETS = 200;
+const MAX_CONNECTIONS = 50;
+const MAX_CONNECTIONS_PER_PRINCIPAL = 2;
+const MAX_CLOCK_SKEW_MS = 6e4;
+var VoiceSessionCreationError = class extends Error {
+	constructor(code, message, status) {
+		super(message);
+		this.code = code;
+		this.status = status;
+		this.name = "VoiceSessionCreationError";
+	}
+};
+function ticketKey(ticket) {
+	return crypto.createHash("sha256").update(ticket).digest("hex");
+}
+function isStreamingProvider(provider) {
+	return Boolean(provider?.streamingAudio && provider.openAudioStream);
+}
+function resolveStreamingStt(config, language) {
+	const sttConfig = mergeSttConfigFromAppConfig(config.tools?.media?.audio, config.tools?.media);
+	for (const entry of resolveSTTProviderChain(sttConfig)) {
+		const plugin = getMediaUnderstandingProvider(entry.id);
+		if (!isStreamingProvider(plugin)) continue;
+		const model = plugin.streamingAudio.defaultModel;
+		return {
+			plugin,
+			model,
+			...entry.apiKey ? { apiKey: entry.apiKey } : {},
+			...entry.baseUrl ? { baseUrl: entry.baseUrl } : {},
+			...entry.headers ? { headers: entry.headers } : {},
+			...language ?? entry.language ? { language: language ?? entry.language } : {},
+			...entry.prompt ? { prompt: entry.prompt } : {},
+			route: {
+				provider: plugin.id,
+				model,
+				managed: plugin.id === "xopc-cloud"
+			}
+		};
+	}
+}
+function resolveStreamingTts(config) {
+	let effective = mergeTtsConfigFromAppConfig(config.messages?.tts);
+	const selection = config.voice?.realtime?.tts;
+	if (selection) {
+		const slice = effective.providers?.[selection.provider] ?? {};
+		const apiKey = (config.tools?.media?.audio?.providers?.[selection.provider])?.apiKey || (selection.provider === "alibaba" ? process.env.DASHSCOPE_API_KEY : void 0) || slice.apiKey;
+		effective = {
+			...effective,
+			enabled: true,
+			provider: selection.provider,
+			managedAuto: false,
+			fallback: {
+				enabled: false,
+				order: []
+			},
+			providers: { [selection.provider]: {
+				...apiKey ? { apiKey } : {},
+				...slice.baseUrl ? { baseUrl: slice.baseUrl } : {},
+				...selection.voice ? { voice: selection.voice } : selection.provider === "alibaba" ? { voice: "Cherry" } : {}
+			} }
+		};
+	}
+	if (!effective.enabled) return void 0;
+	let chain;
+	try {
+		chain = resolveSpeechProviderChain(effective);
+	} catch {
+		return;
+	}
+	for (const provider of chain) {
+		if (typeof provider.plugin.synthesizeStream !== "function") continue;
+		let model;
+		if (provider.plugin.id === "alibaba") model = ALIBABA_REALTIME_TTS_MODEL;
+		else if (provider.plugin.id === "xopc-cloud") {
+			const candidate = typeof provider.providerConfig.model === "string" ? provider.providerConfig.model : void 0;
+			const catalogModel = getModelCatalogStore().getSource("xopc-cloud")?.models.find((entry) => entry.id === candidate);
+			if (!catalogModel?.tts?.streaming || !catalogModel.tts.outputFormats.includes("pcm")) continue;
+			model = candidate;
+		} else continue;
+		if (!model) continue;
+		const rawSlice = effective.providers?.[provider.plugin.id] ?? {};
+		return {
+			provider,
+			config: {
+				...effective,
+				enabled: true,
+				provider: provider.plugin.id,
+				managedAuto: false,
+				trigger: "always",
+				fallback: {
+					enabled: false,
+					order: []
+				},
+				summarization: { enabled: false },
+				modelOverrides: { enabled: false },
+				providers: { [provider.plugin.id]: {
+					...rawSlice,
+					model
+				} }
+			},
+			route: {
+				provider: provider.plugin.id,
+				model,
+				managed: provider.plugin.id === "xopc-cloud"
+			}
+		};
+	}
+}
+function rawText(data) {
+	if (typeof data === "string") return data;
+	if (data instanceof ArrayBuffer) return Buffer.from(data).toString("utf8");
+	if (Array.isArray(data)) return Buffer.concat(data).toString("utf8");
+	return data.toString("utf8");
+}
+var VoiceRealtimeRuntime = class {
+	wss = new WebSocketServer({
+		noServer: true,
+		maxPayload: VOICE_REALTIME_MAX_BINARY_FRAME_BYTES,
+		perMessageDeflate: false
+	});
+	tickets = /* @__PURE__ */ new Map();
+	preauthBudget = createPreauthConnectionBudget();
+	socketsByPrincipal = /* @__PURE__ */ new Map();
+	conversationReservations = /* @__PURE__ */ new Map();
+	closed = false;
+	constructor(options) {
+		this.options = options;
+		this.wss.on("connection", (socket, request) => this.handleConnection(socket, request));
+		this.wss.on("error", (err) => log.error({ err }, "Voice WebSocket server failed"));
+	}
+	async prepareSession(request, now = Date.now()) {
+		const config = structuredClone(this.options.getConfig());
+		this.pruneTickets(now);
+		if (!config.voice?.realtime?.enabled) throw new VoiceSessionCreationError("VOICE_DISABLED", "Realtime voice is disabled", 503);
+		request = request.purpose === "conversation" ? {
+			...request,
+			engine: request.engine ?? config.voice.realtime.defaultEngine
+		} : request;
+		let omni;
+		if (request.engine === "omni") try {
+			omni = await resolveOmniRoute(config);
+		} catch {
+			throw new VoiceSessionCreationError("PROVIDER_UNAVAILABLE", "Natural conversation is unavailable. Check its model, endpoint and credentials.", 503);
+		}
+		if (omni && (!this.options.recordOmniTranscript || !this.options.getConversationContext)) throw new VoiceSessionCreationError("PROVIDER_UNAVAILABLE", "Natural conversation storage is unavailable", 503);
+		const conversationSessionId = request.purpose === "conversation" && request.sessionKey ? await this.options.getSessionIdentity?.(request.sessionKey) : void 0;
+		if (omni && !conversationSessionId) throw new VoiceSessionCreationError("SESSION_NOT_FOUND", "Conversation session was not found", 404);
+		const stt = omni ? void 0 : resolveStreamingStt(config, request.language);
+		if (!omni && !stt) throw new VoiceSessionCreationError("PROVIDER_UNAVAILABLE", "No configured streaming speech-to-text provider is available", 503);
+		const inputMode = "server_vad";
+		if (stt && !stt.plugin.streamingAudio.turnDetection.includes(inputMode)) throw new VoiceSessionCreationError("PROVIDER_UNAVAILABLE", `Streaming speech-to-text provider does not support ${inputMode}`, 503);
+		const tts = request.engine === "agent" ? resolveStreamingTts(config) : void 0;
+		if (request.purpose === "conversation") {
+			if (!request.sessionKey || !await this.options.sessionExists(request.sessionKey)) throw new VoiceSessionCreationError("SESSION_NOT_FOUND", "Conversation session was not found", 404);
+			if (this.options.sessionBusy(request.sessionKey)) throw new VoiceSessionCreationError("SESSION_CONFLICT", "Conversation session already has an active response", 409);
+			if (this.conversationReservations.has(request.sessionKey)) throw new VoiceSessionCreationError("SESSION_CONFLICT", "Conversation session already has a voice connection", 409);
+			if (!omni && !tts) throw new VoiceSessionCreationError("PROVIDER_UNAVAILABLE", "No configured PCM streaming text-to-speech provider is available", 503);
+		}
+		if (this.tickets.size >= MAX_OUTSTANDING_TICKETS) throw new VoiceSessionCreationError("SESSION_LIMIT", "Too many outstanding realtime voice tickets", 429);
+		return {
+			request,
+			config,
+			omni,
+			conversationSessionId,
+			stt,
+			tts,
+			inputMode
+		};
+	}
+	async preflight(request) {
+		await this.prepareSession(request);
+	}
+	async createSession(request, principalId, now = Date.now()) {
+		const { request: resolvedRequest, config, omni, conversationSessionId, stt, tts, inputMode } = await this.prepareSession(request, now);
+		request = resolvedRequest;
+		const sessionId = crypto.randomUUID();
+		const ticket = crypto.randomBytes(32).toString("base64url");
+		const maxSessionMs = request.purpose === "conversation" ? Math.min(config.voice.realtime.maxConversationMs, omni?.route.managed ? 30 * 6e4 : Number.MAX_SAFE_INTEGER) : config.voice.realtime.maxDictationMs;
+		const claim = {
+			conversationSessionId,
+			sessionId,
+			principalId,
+			request,
+			inputMode,
+			idleTimeoutMs: config.voice.realtime.idleTimeoutMs,
+			maxSessionMs,
+			silenceDurationMs: config.voice.realtime.silenceDurationMs,
+			stt,
+			omni,
+			...tts ? { tts } : {},
+			config,
+			createdAt: now,
+			expiresAt: now + TICKET_TTL_MS
+		};
+		this.tickets.set(ticketKey(ticket), claim);
+		if (request.purpose === "conversation" && request.sessionKey) this.conversationReservations.set(request.sessionKey, sessionId);
+		return {
+			sessionId,
+			ticket,
+			ticketExpiresAt: new Date(claim.expiresAt).toISOString(),
+			websocketPath: VOICE_REALTIME_WS_PATH,
+			protocolVersion: 2,
+			purpose: request.purpose,
+			inputMode,
+			bargeIn: config.voice.realtime.bargeIn,
+			inputFormat: {
+				encoding: "pcm_s16le",
+				sampleRate: 16e3,
+				channels: 1
+			},
+			limits: {
+				maxBinaryFrameBytes: VOICE_REALTIME_MAX_BINARY_FRAME_BYTES,
+				maxSessionMs,
+				idleTimeoutMs: claim.idleTimeoutMs
+			},
+			route: omni ? {
+				engine: "omni",
+				omni: omni.route
+			} : tts ? {
+				engine: "agent",
+				stt: stt.route,
+				tts: tts.route
+			} : {
+				engine: "dictation",
+				stt: stt.route
+			}
+		};
+	}
+	handleUpgrade(req, socket, head) {
+		if (new URL(req.url ?? "/", "http://gateway.local").pathname !== "/api/voice/realtime/v2/ws") return false;
+		if (this.closed || this.wss.clients.size >= MAX_CONNECTIONS) {
+			socket.destroy();
+			return true;
+		}
+		const clientIp = req.socket.remoteAddress;
+		if (!this.preauthBudget.acquire(clientIp)) {
+			socket.destroy();
+			return true;
+		}
+		try {
+			this.wss.handleUpgrade(req, socket, head, (ws) => this.wss.emit("connection", ws, req));
+		} catch (error) {
+			this.preauthBudget.release(clientIp);
+			throw error;
+		}
+		return true;
+	}
+	hasConversation(sessionKey) {
+		this.pruneTickets(Date.now());
+		return this.conversationReservations.has(sessionKey);
+	}
+	close() {
+		if (this.closed) return;
+		this.closed = true;
+		for (const client of this.wss.clients) client.close(1001, "Gateway stopping");
+		this.tickets.clear();
+		this.wss.close();
+		this.socketsByPrincipal.clear();
+		this.conversationReservations.clear();
+	}
+	releaseConversationReservation(claim) {
+		const sessionKey = claim.request.sessionKey;
+		if (sessionKey && this.conversationReservations.get(sessionKey) === claim.sessionId) this.conversationReservations.delete(sessionKey);
+	}
+	consumeTicket(ticket, sessionId, now = Date.now()) {
+		const key = ticketKey(ticket);
+		const claim = this.tickets.get(key);
+		this.tickets.delete(key);
+		if (!claim) return void 0;
+		if (claim.expiresAt < now || claim.sessionId !== sessionId) {
+			this.releaseConversationReservation(claim);
+			return;
+		}
+		return claim;
+	}
+	pruneTickets(now) {
+		for (const [key, claim] of this.tickets) if (claim.expiresAt < now) {
+			this.tickets.delete(key);
+			this.releaseConversationReservation(claim);
+		}
+	}
+	handleConnection(socket, request) {
+		const clientIp = request.socket.remoteAddress;
+		let claim;
+		let engine;
+		let ready = false;
+		let seq = 0;
+		let audioSeq = 0;
+		let closed = false;
+		let lastActivityAt = Date.now();
+		const abortController = new AbortController();
+		const seenMessageIds = /* @__PURE__ */ new Set();
+		let context;
+		let unsubscribeMemory;
+		let unsubscribeSession;
+		const send = (type, payload) => {
+			if (!claim || socket.readyState !== WebSocketState.OPEN) return;
+			const event = {
+				protocolVersion: 2,
+				eventId: crypto.randomUUID(),
+				seq: ++seq,
+				type,
+				sentAt: Date.now(),
+				sessionId: claim.sessionId,
+				payload
+			};
+			socket.send(JSON.stringify(event));
+		};
+		const detachPrincipal = () => {
+			if (!claim) return;
+			const sockets = this.socketsByPrincipal.get(claim.principalId);
+			sockets?.delete(socket);
+			if (sockets?.size === 0) this.socketsByPrincipal.delete(claim.principalId);
+		};
+		const shutdown = async (reason, notify) => {
+			if (closed) return;
+			closed = true;
+			unsubscribeMemory?.();
+			unsubscribeSession?.();
+			clearTimeout(startTimer);
+			clearInterval(lifecycleTimer);
+			abortController.abort(reason);
+			try {
+				await engine?.close();
+			} catch (err) {
+				log.error({
+					err,
+					sessionId: claim?.sessionId
+				}, "Voice cleanup failed");
+			}
+			if (notify) send("session.closed", { reason });
+			detachPrincipal();
+			if (claim) this.releaseConversationReservation(claim);
+			this.preauthBudget.release(clientIp);
+			if (socket.readyState === WebSocketState.OPEN) socket.close(1e3, reason.slice(0, 120));
+		};
+		const startTimer = setTimeout(() => socket.close(4401, "Voice session start timeout"), VOICE_REALTIME_START_TIMEOUT_MS);
+		const lifecycleTimer = setInterval(() => {
+			if (!claim) return;
+			if (context?.memory && !context.memory.isCurrent()) {
+				invalidateContext();
+				return;
+			}
+			const now = Date.now();
+			if (now - lastActivityAt > claim.idleTimeoutMs) shutdown("idle_timeout", true);
+			else if (now - claim.createdAt > claim.maxSessionMs) shutdown("session_limit", true);
+		}, 1e3);
+		const invalidateContext = () => {
+			if (closed) return;
+			send("session.error", {
+				code: "CONTEXT_CHANGED",
+				message: "Conversation context changed. Reconnect to use the updated information.",
+				recoverable: false
+			});
+			shutdown("context_changed", true);
+		};
+		const authenticate = async (message) => {
+			const consumed = this.consumeTicket(message.payload.ticket, message.payload.sessionId);
+			if (!consumed) {
+				socket.close(4401, "Invalid voice session ticket");
+				return;
+			}
+			const principalSockets = this.socketsByPrincipal.get(consumed.principalId) ?? /* @__PURE__ */ new Set();
+			if (principalSockets.size >= MAX_CONNECTIONS_PER_PRINCIPAL) {
+				this.releaseConversationReservation(consumed);
+				socket.close(4429, "Voice session concurrency limit");
+				return;
+			}
+			claim = consumed;
+			principalSockets.add(socket);
+			this.socketsByPrincipal.set(consumed.principalId, principalSockets);
+			clearTimeout(startTimer);
+			let setupStage = "context";
+			try {
+				if (consumed.omni) {
+					unsubscribeSession = onUserContextChange((change) => {
+						if (change.kind === "session-reset" && change.id === consumed.request.sessionKey) invalidateContext();
+					});
+					context = await this.options.getConversationContext(consumed.request.sessionKey, consumed.conversationSessionId);
+					if (closed) return;
+					if (context.memory) {
+						unsubscribeMemory = context.memory.subscribe(invalidateContext);
+						if (!context.memory.isCurrent()) {
+							invalidateContext();
+							return;
+						}
+					}
+					consumed.omni = {
+						...consumed.omni,
+						instructions: voiceConversationInstructions(consumed.omni.instructions, context)
+					};
+				}
+				setupStage = "engine";
+				const sendAudio = (responseId, bytes) => {
+					if (socket.bufferedAmount > 1024 * 1024) throw new Error("Voice client audio backpressure limit exceeded");
+					if (!closed && socket.readyState === WebSocketState.OPEN) socket.send(encodeVoiceAudioFrame({
+						responseId,
+						seq: ++audioSeq,
+						audio: bytes
+					}), { binary: true });
+				};
+				engine = consumed.omni ? createOmniVoiceEngine({
+					callId: consumed.sessionId,
+					route: consumed.omni,
+					silenceDurationMs: consumed.silenceDurationMs,
+					bargeIn: consumed.config.voice?.realtime?.bargeIn ?? true,
+					send,
+					sendAudio,
+					record: (entry) => this.options.recordOmniTranscript(consumed.request.sessionKey, consumed.sessionId, entry, consumed.conversationSessionId),
+					onClose: shutdown
+				}) : createAgentVoiceEngine({
+					claim: consumed,
+					runtime: this.options,
+					signal: abortController.signal,
+					send,
+					sendAudio,
+					onClose: shutdown
+				});
+				await engine.start();
+				if (closed) return;
+				ready = true;
+				log.info({
+					sessionId: consumed.sessionId,
+					purpose: consumed.request.purpose,
+					provider: consumed.omni?.route.provider ?? consumed.stt?.route.provider,
+					model: consumed.omni?.route.model ?? consumed.stt?.route.model,
+					latencyMs: Date.now() - consumed.createdAt
+				}, "Realtime voice session ready");
+				send("session.ready", {
+					purpose: consumed.request.purpose,
+					inputMode: consumed.inputMode,
+					inputFormat: {
+						encoding: "pcm_s16le",
+						sampleRate: 16e3,
+						channels: 1
+					},
+					route: consumed.omni ? {
+						engine: "omni",
+						omni: consumed.omni.route
+					} : consumed.tts ? {
+						engine: "agent",
+						stt: consumed.stt.route,
+						tts: consumed.tts.route
+					} : {
+						engine: "dictation",
+						stt: consumed.stt.route
+					},
+					heartbeatIntervalMs: VOICE_REALTIME_HEARTBEAT_INTERVAL_MS
+				});
+			} catch (error) {
+				if (closed) return;
+				log.warn({
+					err: error,
+					sessionId: consumed.sessionId,
+					provider: consumed.omni?.route.provider ?? consumed.stt?.route.provider
+				}, "Realtime voice setup failed");
+				send("session.error", {
+					code: "PROVIDER_UNAVAILABLE",
+					message: setupStage === "context" ? "Conversation context could not be restored. Check voice and agent instructions." : "Voice engine is unavailable",
+					recoverable: false
+				});
+				await shutdown("provider_unavailable", true);
+			}
+		};
+		const clientMetrics = /* @__PURE__ */ new Set();
+		socket.on("message", (data, isBinary) => {
+			lastActivityAt = Date.now();
+			if (isBinary) {
+				if (!claim) {
+					socket.close(4401, "Voice session is not ready");
+					return;
+				}
+				const bytes = data instanceof ArrayBuffer ? new Uint8Array(data) : Array.isArray(data) ? Buffer.concat(data) : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+				if (bytes.byteLength === 0 || bytes.byteLength % 2 !== 0 || bytes.byteLength > 65536) {
+					send("session.error", {
+						code: "INVALID_AUDIO",
+						message: "Invalid PCM audio frame",
+						recoverable: false
+					});
+					shutdown("invalid_audio", true);
+					return;
+				}
+				if (!ready || !engine) {
+					socket.close(4401, "Voice session is not ready");
+					return;
+				}
+				try {
+					engine.appendAudio(bytes);
+				} catch (error) {
+					log.warn({
+						err: error,
+						sessionId: claim.sessionId
+					}, "Realtime voice engine rejected audio");
+					send("session.error", {
+						code: "AUDIO_BACKPRESSURE",
+						message: "Audio stream cannot keep up",
+						recoverable: false
+					});
+					shutdown("audio_backpressure", true);
+				}
+				return;
+			}
+			let message;
+			try {
+				message = parseVoiceClientMessage(parseVoiceClientJsonFrame(rawText(data)));
+				if (Math.abs(Date.now() - message.sentAt) > MAX_CLOCK_SKEW_MS) throw new Error("Clock skew");
+				if (seenMessageIds.has(message.messageId)) return;
+				if (seenMessageIds.size >= 256) seenMessageIds.clear();
+				seenMessageIds.add(message.messageId);
+			} catch (error) {
+				log.debug({ err: error }, "Voice client sent an invalid control frame");
+				socket.close(4400, "Invalid voice protocol frame");
+				return;
+			}
+			if (!claim) {
+				if (message.type !== "session.start") {
+					socket.close(4401, "First voice message must be session.start");
+					return;
+				}
+				authenticate(message);
+				return;
+			}
+			if (message.type === "session.ping") send("session.pong", {});
+			else if (message.type === "session.stop") shutdown(message.payload.reason, true);
+			else if (message.type === "session.metric") {
+				const { responseId, metric, durationMs } = message.payload;
+				const key = `${responseId}:${metric}`;
+				if (!clientMetrics.has(key) && clientMetrics.size < 512) {
+					clientMetrics.add(key);
+					log.info({
+						sessionId: claim.sessionId,
+						responseId,
+						metric,
+						durationMs,
+						source: "client",
+						engine: claim.request.engine
+					}, "Voice client timing sample");
+				}
+			} else if (message.type === "response.cancel") {
+				if (!engine?.cancel(message.payload.responseId, "client_cancelled")) send("session.error", {
+					code: "NO_ACTIVE_RESPONSE",
+					message: "No matching response is active",
+					recoverable: true
+				});
+			} else if (message.type === "response.audio.played") try {
+				engine?.acknowledge(message.payload.responseId, message.payload.playedBytes);
+			} catch {
+				socket.close(4400, "Invalid playback acknowledgement");
+			}
+			else if (message.type === "input.mute") {
+				if (!ready || !engine || claim.request.purpose !== "conversation") return;
+				Promise.resolve().then(() => engine.setInputMuted(message.payload.muted)).catch((error) => {
+					log.warn({
+						err: error,
+						sessionId: claim?.sessionId
+					}, "Voice input reset failed");
+					send("session.error", {
+						code: "PROVIDER_ERROR",
+						message: "Could not resume microphone input",
+						recoverable: false
+					});
+					shutdown("provider_error", true);
+				});
+			} else if (message.type === "input.commit") {
+				if (!ready || !engine || claim.request.purpose !== "dictation") {
+					send("session.error", {
+						code: "INVALID_STATE",
+						message: "Input cannot be committed",
+						recoverable: true
+					});
+					return;
+				}
+				ready = false;
+				engine.commit().catch(async (error) => {
+					log.warn({
+						err: error,
+						sessionId: claim?.sessionId
+					}, "Voice input commit failed");
+					send("session.error", {
+						code: "PROVIDER_ERROR",
+						message: "Voice input commit failed",
+						recoverable: false
+					});
+					await shutdown("provider_error", true);
+				});
+			} else send("session.error", {
+				code: "INVALID_STATE",
+				message: "Voice session is already started",
+				recoverable: true
+			});
+		});
+		socket.once("close", () => void shutdown("disconnected", false));
+		socket.once("error", (err) => log.debug({ err }, "Voice client socket failed"));
+	}
+};
+//#endregion
+export { VoiceRealtimeRuntime, VoiceSessionCreationError, resolveStreamingStt, resolveStreamingTts };

package/dist/src/voice/realtime/speakable-segmenter.d.ts

@@ -0,0 +1,10 @@
+/** Streams spoken prose, keeping code and link targets out of synthesis. */
+export declare class SpeakableSegmenter {
+    private readonly maxCharacters;
+    private pending;
+    private fence;
+    constructor(maxCharacters?: number);
+    push(delta: string): string[];
+    flush(): string[];
+    private toSpeech;
+}