@xopcai/xopc 0.0.242 → 0.0.244

package/dist/src/gateway/hono/routes/lazy-fallback.js

@@ -3,6 +3,16 @@ import { APP_LAZY_ROUTE_BUNDLES, AUTHENTICATED_LAZY_ROUTE_BUNDLES, findAuthentic
 import { Hono } from "hono";
 //#region src/gateway/hono/routes/lazy-fallback.ts
 const log = createGatewayRouteLogger("LazyRoutes");
+const FORWARDED_VARIABLES = Symbol("gateway.forwardedVariables");
+function createSubApp() {
+	const sub = new Hono();
+	sub.use(async (c, next) => {
+		const variables = c.env?.[FORWARDED_VARIABLES];
+		for (const [key, value] of Object.entries(variables ?? {})) c.set(key, value);
+		await next();
+	});
+	return sub;
+}
 const authenticatedSubApps = /* @__PURE__ */ new Map();
 const appSubApps = /* @__PURE__ */ new Map();
 const authenticatedLoadPromises = /* @__PURE__ */ new Map();
@@ -28,7 +38,7 @@ async function ensureAuthenticatedLazyBundle(bundleId, deps) {
 		if (!bundle) return null;
 		pending = (async () => {
 			const mod = await bundle.load();
-			const sub = new Hono();
+			const sub = createSubApp();
 			mod.register(sub, deps);
 			authenticatedSubApps.set(bundleId, sub);
 			authenticatedLoadPromises.delete(bundleId);
@@ -47,7 +57,7 @@ async function ensureAppLazyBundle(bundleId, params) {
 		if (!bundle) return null;
 		pending = (async () => {
 			const mod = await bundle.load();
-			const sub = new Hono();
+			const sub = createSubApp();
 			if (mod.registerOnApp) mod.registerOnApp(sub, params.service);
 			appSubApps.set(bundleId, sub);
 			appLoadPromises.delete(bundleId);
@@ -58,10 +68,14 @@ async function ensureAppLazyBundle(bundleId, params) {
 	return pending;
 }
 async function forwardToSubApp(c, sub) {
+	const env = {
+		...c.env,
+		[FORWARDED_VARIABLES]: { ...c.var }
+	};
 	try {
-		return await sub.fetch(c.req.raw, c.env, c.executionCtx);
+		return await sub.fetch(c.req.raw, env, c.executionCtx);
 	} catch (error) {
-		if (error instanceof Error && error.message.includes("ExecutionContext")) return sub.fetch(c.req.raw, c.env);
+		if (error instanceof Error && error.message.includes("ExecutionContext")) return sub.fetch(c.req.raw, env);
 		logRouteError(log, c, error, "gateway.route.lazy_forward");
 		throw error;
 	}

package/dist/src/gateway/hono/routes/models.js

@@ -1,5 +1,5 @@
-import { init_resolve_config_value, testApiKeyResolution } from "../../../config/resolve-config-value.js";
 import { resolveModelsJsonPath } from "../../../config/paths.js";
+import { init_resolve_config_value, testApiKeyResolution } from "../../../config/resolve-config-value.js";
 import { init_models_json, loadModelsJson, saveModelsJson, validateModelsConfig } from "../../../config/models-json.js";
 import { getModelCatalogStore, init_model_catalog_store } from "../../../providers/model-catalog-store.js";
 import { getModelRegistry } from "../../../providers/model-registry.js";

package/dist/src/gateway/hono/routes/session-input-handler.js

@@ -73,6 +73,13 @@ async function submitSessionInput(c, deps, sessionKey) {
 		}
 	}, 400);
 	return withModelConfigLock(sessionKey, async () => {
+		if (deps.service.voiceRealtime?.hasConversation(sessionKey)) return c.json({
+			ok: false,
+			error: {
+				code: "SESSION_BUSY",
+				message: "End the voice call before sending text"
+			}
+		}, 409);
 		const selection = body.configVersion !== void 0 ? await deps.service.sessions.getAgentConfig(sessionKey) : void 0;
 		if (selection) {
 			if (!Number.isSafeInteger(body.configVersion) || selection.configVersion !== body.configVersion || !selection.fixedModel) return c.json({
@@ -181,6 +188,13 @@ async function replaceLatestSessionTurn(c, deps, sessionKey, targetTurnId) {
 		}
 	}, 401);
 	return withModelConfigLock(sessionKey, async () => {
+		if (deps.service.voiceRealtime?.hasConversation(sessionKey)) return c.json({
+			ok: false,
+			error: {
+				code: "SESSION_BUSY",
+				message: "End the voice call before editing a turn"
+			}
+		}, 409);
 		const selection = body.configVersion !== void 0 ? await deps.service.sessions.getAgentConfig(sessionKey) : void 0;
 		if (selection) {
 			if (!Number.isSafeInteger(body.configVersion) || selection.configVersion !== body.configVersion || !selection.fixedModel) return c.json({

package/dist/src/gateway/hono/routes/sessions.js

@@ -785,6 +785,7 @@ function registerSessionsRoutes(authenticated, deps) {
 	});
 	authenticated.delete("/api/sessions/:key/messages", async (c) => {
 		const key = c.req.param("key");
+		if (service.voiceRealtime?.hasConversation(key)) return c.json({ error: "End the voice call before editing its history" }, 409);
 		const body = await c.req.json().catch(() => ({}));
 		const userRoundIndex = typeof body.userRoundIndex === "number" ? body.userRoundIndex : void 0;
 		if (userRoundIndex === void 0 || !Number.isInteger(userRoundIndex) || userRoundIndex < 0) return c.json({ error: "userRoundIndex must be a non-negative integer" }, 400);
@@ -805,6 +806,7 @@ function registerSessionsRoutes(authenticated, deps) {
 		const blocked = ensureGatewayReadyForSessions(c, service, "sessions.reset");
 		if (blocked) return blocked;
 		const key = c.req.param("key");
+		if (service.voiceRealtime?.hasConversation(key)) return c.json({ error: "End the voice call before resetting the session" }, 409);
 		const result = await service.sessions.reset(key);
 		if (result.ok === false) {
 			const status = result.error === "Session not found" ? 404 : 400;
@@ -824,6 +826,7 @@ function registerSessionsRoutes(authenticated, deps) {
 	});
 	authenticated.delete("/api/sessions/:key", async (c) => {
 		const key = c.req.param("key");
+		if (service.voiceRealtime?.hasConversation(key)) return c.json({ error: "End the voice call before deleting the session" }, 409);
 		if (environments.get(key) && service.getActiveWebchatRunId(key)) return c.json({
 			ok: false,
 			error: "Stop the active session run before deleting its environment"

package/dist/src/gateway/hono/routes/voice.js

@@ -2,7 +2,7 @@ import { getDefaultModelSync, init_providers, resolveModel } from "../../../prov
 import { completeWithResolvedCredentials } from "../../../providers/model-call.js";
 import { resolveTtsProviderConfigSlice } from "../../../voice/tts/config-slice.js";
 import { isTTSAvailable, resolveSpeechProvider } from "../../../voice/tts/factory.js";
-import { speak } from "../../../voice/tts/speak-core.js";
+import { speak, speakStream } from "../../../voice/tts/speak-core.js";
 import { mergeTtsConfigFromAppConfig } from "../../../voice/tts/merge-config.js";
 import { getLocalVoiceRuntimeClient } from "../../../voice/local/runtime-client.js";
 import { getAudioDecoderStatus } from "../../../voice/audio/normalize.js";
@@ -13,6 +13,11 @@ import { mergeSttConfigFromAppConfig } from "../../../voice/stt/config.js";
 import "../../../voice/stt/index.js";
 import { listSttProvidersForApi } from "../../../voice/stt/list-providers.js";
 import "../../../voice/tts/index.js";
+import { withModelConfigLock } from "../../../session/model-config-lock.js";
+import { createVoiceSessionRequestSchema } from "../../../../packages/realtime-protocol/src/voice.js";
+import { resolveOmniRoute } from "../../../voice/realtime/omniRoute.js";
+import { VoiceSessionCreationError, resolveStreamingStt, resolveStreamingTts } from "../../../voice/realtime/runtime.js";
+import { getGatewayPrincipal } from "../../security/gateway-principal.js";
 import { createGatewayRouteLogger } from "../lib/route-logger.js";
 import { listLocalVoiceModelStatuses, removeLocalVoiceModel, startLocalVoiceModelInstall } from "../../../voice/local/model-manager.js";
 import { listTtsProvidersForApi } from "../../../voice/tts/list-providers.js";
@@ -107,6 +112,134 @@ async function refineTranscript(raw, config, signal) {
 }
 function registerVoiceRoutes(authenticated, deps) {
 	const { service, strictRateLimitMiddleware } = deps;
+	authenticated.get("/api/voice/realtime/status", async (c) => {
+		const config = service.currentConfig;
+		const stt = resolveStreamingStt(config);
+		const tts = resolveStreamingTts(config);
+		const omni = await resolveOmniRoute(config).catch(() => null);
+		const enabled = config.voice?.realtime?.enabled === true;
+		const availability = (configured) => ({
+			available: enabled && configured,
+			...!enabled ? { reasonCode: "VOICE_DISABLED" } : !configured ? { reasonCode: "PROVIDER_UNAVAILABLE" } : {}
+		});
+		return c.json({
+			ok: true,
+			payload: {
+				enabled,
+				capabilities: {
+					dictation: availability(Boolean(stt)),
+					agent: availability(Boolean(stt && tts)),
+					omni: availability(Boolean(omni)),
+					languages: ["zh", "en"],
+					bargeIn: config.voice?.realtime?.bargeIn ?? true
+				},
+				defaultEngine: config.voice?.realtime?.defaultEngine ?? "agent",
+				omni: omni?.route ?? null,
+				stt: stt?.route ?? null,
+				tts: tts ? {
+					...tts.route,
+					voice: tts.config.providers?.[tts.route.provider]?.voice ?? tts.provider.providerConfig.voice
+				} : null
+			}
+		});
+	});
+	authenticated.post("/api/voice/realtime/preview", strictRateLimitMiddleware, async (c) => {
+		const config = service.currentConfig;
+		const route = resolveStreamingTts(config);
+		if (!route) return c.json({
+			ok: false,
+			error: { message: "Realtime voice output is not configured" }
+		}, 503);
+		const controller = new AbortController();
+		const signal = AbortSignal.any([
+			c.req.raw.signal,
+			controller.signal,
+			AbortSignal.timeout(2e4)
+		]);
+		let result;
+		let reader;
+		try {
+			result = await speakStream(config.voice?.language === "en" ? "Hello, I am your assistant. You can talk to me naturally." : "你好，我是你的助手。现在可以和我自然对话了。", route.config, {
+				signal,
+				appConfig: config,
+				allowFallback: false,
+				parseDirectives: false
+			});
+			if (result.outputFormat !== "pcm") throw new Error("Realtime preview requires PCM audio");
+			reader = result.audioStream.getReader();
+			const chunks = [];
+			let size = 0;
+			while (true) {
+				signal.throwIfAborted();
+				const { value, done } = await reader.read();
+				if (done) break;
+				size += value.byteLength;
+				if (size > 96e4) throw new Error("Realtime preview audio exceeds the limit");
+				chunks.push(value);
+			}
+			if (size === 0 || size % 2 !== 0) throw new Error("Realtime preview returned invalid audio");
+			return c.json({
+				ok: true,
+				payload: {
+					audio: Buffer.concat(chunks).toString("base64"),
+					sampleRate: 24e3
+				}
+			});
+		} catch (error) {
+			log.warn({
+				err: error,
+				provider: route.route.provider
+			}, "Realtime voice preview failed");
+			return c.json({
+				ok: false,
+				error: { message: "Voice preview failed. Check the provider credentials and try again." }
+			}, 502);
+		} finally {
+			controller.abort();
+			await reader?.cancel().catch(() => void 0);
+			reader?.releaseLock();
+			await result?.release();
+		}
+	});
+	for (const action of ["preflight", "sessions"]) authenticated.post(`/api/voice/realtime/${action}`, strictRateLimitMiddleware, async (c) => {
+		const parsed = createVoiceSessionRequestSchema.safeParse(await c.req.json().catch(() => null));
+		if (!parsed.success) return c.json({
+			ok: false,
+			error: {
+				code: "INVALID_REQUEST",
+				message: "Invalid realtime voice session request"
+			}
+		}, 400);
+		try {
+			if (action === "preflight") {
+				await service.voiceRealtime.preflight(parsed.data);
+				return c.json({ ok: true });
+			}
+			const principal = getGatewayPrincipal(c);
+			const create = () => service.voiceRealtime.createSession(parsed.data, principal.principalId);
+			const payload = parsed.data.sessionKey ? await withModelConfigLock(parsed.data.sessionKey, create) : await create();
+			return c.json({
+				ok: true,
+				payload
+			});
+		} catch (error) {
+			const message = error instanceof Error ? error.message : "Realtime voice is unavailable";
+			if (error instanceof VoiceSessionCreationError) return c.json({
+				ok: false,
+				error: {
+					code: error.code,
+					message
+				}
+			}, error.status);
+			return c.json({
+				ok: false,
+				error: {
+					code: "PROVIDER_UNAVAILABLE",
+					message
+				}
+			}, 503);
+		}
+	});
 	authenticated.post("/api/voice/language", strictRateLimitMiddleware, async (c) => {
 		let body = {};
 		try {
@@ -153,7 +286,11 @@ function registerVoiceRoutes(authenticated, deps) {
 			error: { message: "provider and model are required" }
 		}, 400);
 		const config = service.currentConfig;
-		const baseConfig = mergeTtsConfigFromAppConfig(config.messages?.tts);
+		const baseConfig = c.req.query("purpose") === "realtime" ? resolveStreamingTts(config)?.config : mergeTtsConfigFromAppConfig(config.messages?.tts);
+		if (!baseConfig) return c.json({
+			ok: true,
+			payload: { voices: [] }
+		});
 		const resolved = resolveSpeechProvider(provider, {
 			...baseConfig,
 			provider,

package/dist/src/gateway/run-loop.js

@@ -1,6 +1,6 @@
 import { loadConfig } from "../config/loader.js";
-import "../config/index.js";
 import { isRestartEnabled } from "../config/commands.flags.js";
+import "../config/index.js";
 import { runBootstrapMigrationsSync } from "../migrations/runner.js";
 import { restartGatewayProcessWithFreshPid } from "./respawn.js";
 import { acquireGatewayLock } from "./lock.js";

package/dist/src/gateway/server.js

@@ -68,6 +68,7 @@ var GatewayServer = class {
 			});
 			inner.on("upgrade", (req, socket, head) => {
 				try {
+					if (this.service.voiceRealtime.handleUpgrade(req, socket, head)) return;
 					if (this.service.realtime.handleUpgrade(req, socket, head)) return;
 					if (handleSiteShareUpgrade(this.service, req, socket, head)) return;
 					socket.destroy();

package/dist/src/gateway/service/agent-runner.d.ts

@@ -71,6 +71,7 @@ export declare class GatewayAgentRunner {
         signal?: AbortSignal;
         runId?: string;
         sourceContexts?: AgentSourceContext[];
+        presentation?: 'voice';
     }): AsyncGenerator<{
         type: string;
         [key: string]: unknown;

package/dist/src/gateway/service/agent-runner.js

@@ -128,12 +128,7 @@ var GatewayAgentRunner = class {
 			completeRealtimeTopic: this.opts.completeRealtimeTopic
 		}, message, channel, chatId, origin, attachments, thinking, runOptions);
 		try {
-			let step = await iter.next();
-			while (!step.done) {
-				yield step.value;
-				step = await iter.next();
-			}
-			return step.value;
+			return yield* iter;
 		} finally {
 			if (trackedRunId) {
 				this.resolveRunCompletions.get(trackedRunId)?.();

package/dist/src/gateway/service/config-coordinator.js

@@ -1,7 +1,7 @@
 import { createLogger } from "../../utils/logger/index.js";
 import { init_logger } from "../../utils/logger.js";
-import { getModelRegistry } from "../../providers/model-registry.js";
 import { loadConfig, saveConfig } from "../../config/loader.js";
+import { getModelRegistry } from "../../providers/model-registry.js";
 import { init_providers } from "../../providers/index.js";
 import { disposeAllSessionMcpRuntimes } from "../../agent/mcp/bundle-mcp-runtime.js";
 import { reloadImageGenerationProviders } from "../../agent/image/generation/provider-registry.js";

package/dist/src/gateway/service/run-gateway-agent.d.ts

@@ -25,6 +25,7 @@ export declare function runGatewayAgent(deps: RunGatewayAgentDeps, message: stri
     signal?: AbortSignal;
     runId?: string;
     sourceContexts?: AgentSourceContext[];
+    presentation?: 'voice';
 }): AsyncGenerator<RunGatewayAgentYield, {
     status: string;
     summary: string;

package/dist/src/gateway/service/run-gateway-agent.js

@@ -131,7 +131,8 @@ async function* runGatewayAgent(deps, message, channel, chatId, origin, attachme
 				const eventStream = agentService.turnDispatcher.processDirectStreaming(stampedMessage, sessionKey, origin, prepared, thinking, {
 					signal: mergedSignal,
 					runId,
-					sourceContexts: runOptions?.sourceContexts
+					sourceContexts: runOptions?.sourceContexts,
+					presentation: runOptions?.presentation
 				});
 				const mappedEvents = (async function* () {
 					for await (const event of eventStream) yield* mapper.map(event);
@@ -150,6 +151,18 @@ async function* runGatewayAgent(deps, message, channel, chatId, origin, attachme
 					summary: mergedSignal.aborted ? "Interrupted" : "Message processed successfully"
 				};
 			} catch (error) {
+				if (mergedSignal.aborted) {
+					terminalStatus = "cancelled";
+					taskRunStatus = "cancelled";
+					taskRunSummary = "Interrupted";
+					yield* emitAndYield(mapper.end("cancelled", "Interrupted"));
+					completeRealtimeTopic(`run:${runId}`);
+					runTopicCompleted = true;
+					return {
+						status: "aborted",
+						summary: "Interrupted"
+					};
+				}
 				const em = error instanceof Error ? error.message : String(error);
 				log.error({
 					err: error,
@@ -173,6 +186,11 @@ async function* runGatewayAgent(deps, message, channel, chatId, origin, attachme
 					summary: streamError
 				};
 			} finally {
+				if (mergedSignal.aborted) {
+					terminalStatus = "cancelled";
+					taskRunStatus = "cancelled";
+					taskRunSummary = "Interrupted";
+				}
 				if (registeredActiveWebchatRun && activeWebchatRunBySession.get(sessionKey) === runId) {
 					activeWebchatRunBySession.delete(sessionKey);
 					publishRealtime("sessions", "run.completed", {

package/dist/src/gateway/service.d.ts

@@ -19,6 +19,7 @@ import { GatewaySessionsApi } from './service/sessions-api.js';
 import { GatewayMarketplaceService } from './service/marketplace-service.js';
 import { GatewayAgentRunner } from './service/agent-runner.js';
 import { RealtimeRuntime } from '../realtime/runtime.js';
+import { VoiceRealtimeRuntime } from '../voice/realtime/runtime.js';
 import type { GatewayServiceConfig } from './service/types.js';
 import { type GatewayReadinessSnapshot } from './startup-readiness.js';
 import { type ConnectorLearningCoordinator } from '../connectors/learning-coordinator.js';
@@ -51,6 +52,7 @@ export declare class GatewayService {
     private auth;
     readonly endpointTools: EndpointToolRuntime;
     readonly realtime: RealtimeRuntime;
+    readonly voiceRealtime: VoiceRealtimeRuntime;
     getConfig(): Config;
     getModelCatalogSync(): ModelCatalogSyncService;
     get browserRecipes(): BrowserRecipeService;

package/dist/src/gateway/service.js

@@ -6,17 +6,20 @@ import { getRuntimeLogStats } from "../utils/logger/stats.js";
 import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
 import { closeXopcDatabase, openXopcDatabase } from "../storage/sqlite/connection.js";
+import { notifyUserContextChange } from "../user-context/changes.js";
+import { getSessionConfig } from "../storage/sqlite/config-repository.js";
 import "../storage/sqlite/index.js";
 import { getAgentDefaultModelRef, init_schema } from "../config/schema.js";
 import { init_agent_scope, listAgentEntries, normalizeAgentId, resolveDefaultAgentId } from "../agent/agent-scope.js";
+import { buildSessionKey, init_session_key, sanitizeSegment } from "../routing/session-key.js";
+import { getDefaultAgentId, init_resolve_route } from "../routing/resolve-route.js";
+import { resolveEffectiveAgentProfileForSession } from "../config/agent-profile.js";
 import { init_paths, resolveAgentDir, resolveConfigPath, resolveExtensionsDir } from "../config/paths.js";
-import { prewarmModelRegistry } from "../providers/model-registry.js";
 import { areExtensionsGloballyDisabled } from "../extensions/discover-extensions.js";
 import { buildExtensionMetadataSnapshot } from "../extensions/extension-metadata-snapshot.js";
 import { buildChannelCatalogForConfig, buildChannelCatalogFromSnapshot, init_channel_catalog_service } from "../channels/catalog/channel-catalog-service.js";
 import { loadConfig, saveConfig } from "../config/loader.js";
-import { buildSessionKey, init_session_key, sanitizeSegment } from "../routing/session-key.js";
-import { getDefaultAgentId, init_resolve_route } from "../routing/resolve-route.js";
+import { prewarmModelRegistry } from "../providers/model-registry.js";
 import { init_providers, providers_exports } from "../providers/index.js";
 import { TaskRepository } from "../tasks/task-repository.js";
 import { TaskConversationRepository } from "../tasks/task-conversation-repository.js";
@@ -56,18 +59,20 @@ import "../knowledge/index.js";
 import { startConnectorLearningCoordinator } from "../connectors/learning-coordinator.js";
 import { ConnectedSourceChangePublisher } from "../connectors/source-change-publisher.js";
 import { startConnectorSupervisor } from "../connectors/supervisor.js";
+import { EndpointToolRuntime } from "../endpoint-tools/runtime.js";
+import "../endpoint-tools/index.js";
 import { DiscussionService } from "../discussions/service.js";
 import { DiscussionOrganizer } from "../discussions/organizer.js";
 import { DiscussionLiveWorker } from "../discussions/live-worker.js";
 import { DiscussionOrganizerWorker } from "../discussions/organizer-worker.js";
 import { DiscussionSealer } from "../discussions/sealer.js";
 import "../discussions/index.js";
-import { EndpointToolRuntime } from "../endpoint-tools/runtime.js";
-import "../endpoint-tools/index.js";
 import { assertGatewayAuthConfigured, extractToken, resolveGatewayAuth, validateToken } from "./auth.js";
 import { prepareAutomationAgentSession } from "./automation-agent-session.js";
 import { stopTailscaleExposure } from "./tailscale-lifecycle.js";
 import { reconcileDreamingAutomations } from "./dreaming-automation-reconciler.js";
+import { buildVoiceMemoryContext } from "../voice/realtime/memory-context.js";
+import { voiceMemoryBudget } from "../voice/realtime/conversation-context.js";
 import { NotesStore } from "../notes/store.js";
 import { buildNoteAgentContext } from "../notes/agent-context.js";
 import { NotesService } from "../notes/service.js";
@@ -114,6 +119,7 @@ import { GatewayMarketplaceService } from "./service/marketplace-service.js";
 import { GatewayConfigCoordinator } from "./service/config-coordinator.js";
 import { GatewayAgentRunner } from "./service/agent-runner.js";
 import { RealtimeRuntime } from "../realtime/runtime.js";
+import { VoiceRealtimeRuntime } from "../voice/realtime/runtime.js";
 import { GatewayReadiness } from "./startup-readiness.js";
 import { createGatewayStartupTrace } from "./startup-trace.js";
 import { applyAutomaticVoiceLanguage, inferProductLanguageFromEnvironment, initializeVoiceDefaults } from "../voice/language-profile.js";
@@ -158,6 +164,71 @@ var GatewayService = class {
 	auth;
 	endpointTools = new EndpointToolRuntime();
 	realtime = new RealtimeRuntime(this.endpointTools);
+	voiceRealtime = new VoiceRealtimeRuntime({
+		getConversationContext: async (sessionKey, expectedSessionId) => {
+			if ((await this.sessionIndex.getSessionMetadata(sessionKey))?.sessionId !== expectedSessionId) throw new Error("Conversation changed before voice connection");
+			const messages = await this.sessionIndex.loadMessages(sessionKey);
+			const after = await this.sessionIndex.getSessionMetadata(sessionKey);
+			if (after?.sessionId !== expectedSessionId) throw new Error("Conversation changed while loading voice context");
+			const profile = resolveEffectiveAgentProfileForSession(this.config, sessionKey);
+			const context = {
+				identity: [`Your name is ${profile.config.profile?.name ?? profile.agentId}.`, profile.customInstructions ?? ""].join("\n"),
+				history: messages
+			};
+			const memory = buildVoiceMemoryContext({
+				getConfig: () => this.config,
+				sessionKey,
+				workspaceId: getSessionConfig(sessionKey)?.workingDirectoryOverride ?? profile.resolvedWorkspacePath,
+				projectId: after?.projectId,
+				history: messages,
+				maxChars: voiceMemoryBudget(this.config.voice.realtime.omni.instructions, context)
+			});
+			return {
+				...context,
+				...memory ? { memory } : {}
+			};
+		},
+		getSessionIdentity: async (sessionKey) => (await this.sessionIndex.getSessionMetadata(sessionKey))?.sessionId,
+		recordOmniTranscript: async (sessionKey, callId, entry, expectedSessionId) => {
+			await this.sessionIndex.appendTranscriptCustomMessageEntry(sessionKey, {
+				expectedSessionId,
+				customType: "voice_omni_transcript",
+				content: entry.text,
+				display: true,
+				details: {
+					callId,
+					itemId: entry.itemId,
+					role: entry.role,
+					interrupted: entry.interrupted,
+					engine: "omni"
+				}
+			});
+			this.emit("session.transcript_updated", { key: sessionKey });
+		},
+		getConfig: () => this.config,
+		sessionExists: async (sessionKey) => Boolean(await this.sessionIndex.getSessionMetadata(sessionKey)),
+		sessionBusy: (sessionKey) => Boolean(this.agentRunner.getActiveRunId(sessionKey)) || this.agentRunner.inputs.snapshot(sessionKey).inputs.some((input) => input.status === "queued"),
+		recordInterruption: (entry) => this.sessionIndex.appendTranscriptContextEntry(entry.sessionKey, {
+			text: "Voice response was interrupted before playback completed.",
+			data: {
+				type: "voice_response_interrupted",
+				responseId: entry.responseId,
+				reason: entry.reason,
+				generatedCharacters: entry.generatedCharacters,
+				interruptedDuring: entry.interruptedDuring
+			}
+		}),
+		runAgent: (text, sessionKey, signal) => {
+			if (this.agentRunner.hasActiveRun(sessionKey)) throw new Error("Conversation session already has an active response");
+			return this.agentRunner.runAgent(text, "webchat", sessionKey, {
+				type: "channel",
+				channel: "webchat"
+			}, void 0, void 0, {
+				signal,
+				presentation: "voice"
+			});
+		}
+	});
 	getConfig() {
 		return this.config;
 	}
@@ -1097,6 +1168,7 @@ var GatewayService = class {
 		this.endpointTools.close();
 		await this.sideChats.disposeAll();
 		this.realtime.close();
+		this.voiceRealtime.close();
 		await this.proactiveWorker.stop();
 		await this.discussionWorker.stop();
 		await this.discussionSealer.stop();
@@ -1543,6 +1615,7 @@ var GatewayService = class {
 		});
 	}
 	emit(type, payload) {
+		if (type === "config.reload") notifyUserContextChange({ kind: "policy" });
 		this.realtime.broker.publish("gateway", type, payload);
 		this.createNotificationService().handleGatewayEvent(type, payload);
 	}

package/dist/src/heartbeat/index.js

@@ -1,6 +1,6 @@
 import { DEFAULT_ACK_MAX_CHARS, HEARTBEAT_OK, NO_REPLY, shouldSilence, stripHeartbeatToken } from "./tokens.js";
+import { appendCronEventLines } from "./event-prompt.js";
 import { isWithinActiveHours } from "./active-hours.js";
 import { isHeartbeatContentEmpty } from "./content-check.js";
-import { appendCronEventLines } from "./event-prompt.js";
 import { createHeartbeatWake } from "./wake.js";
 export { DEFAULT_ACK_MAX_CHARS, HEARTBEAT_OK, NO_REPLY, appendCronEventLines, createHeartbeatWake, isHeartbeatContentEmpty, isWithinActiveHours, shouldSilence, stripHeartbeatToken };

package/dist/src/media-understanding/types.d.ts

@@ -112,6 +112,65 @@ export interface AudioTranscriptionResult {
     /** Audio duration in seconds, when reported. */
     durationSeconds?: number;
 }
+export interface PcmAudioFormat {
+    encoding: 'pcm_s16le';
+    sampleRate: 16_000 | 24_000;
+    channels: 1;
+}
+export interface StreamingSttCapabilities {
+    inputSampleRates: readonly number[];
+    turnDetection: readonly 'server_vad'[];
+    defaultModel: string;
+    models: readonly string[];
+}
+export type StreamingSttEvent = {
+    type: 'ready';
+} | {
+    type: 'speech_started';
+    utteranceId: string;
+} | {
+    type: 'speech_stopped';
+    utteranceId: string;
+} | {
+    type: 'transcript_delta';
+    utteranceId: string;
+    revision: number;
+    text: string;
+} | {
+    type: 'transcript_final';
+    utteranceId: string;
+    revision: number;
+    text: string;
+    language?: string;
+} | {
+    type: 'usage';
+    inputAudioMs: number;
+} | {
+    type: 'error';
+    error: Error;
+};
+export interface StreamingSttOpenRequest {
+    model: string;
+    inputFormat: PcmAudioFormat;
+    apiKey?: string;
+    baseUrl?: string;
+    headers?: Record<string, string>;
+    language?: string;
+    prompt?: string;
+    turnDetection: {
+        mode: 'server_vad';
+        silenceDurationMs: number;
+    };
+    timeoutMs: number;
+    signal: AbortSignal;
+    onEvent: (event: StreamingSttEvent) => void;
+}
+export interface StreamingSttSession {
+    appendAudio(chunk: Uint8Array): void;
+    commit(): Promise<void>;
+    close(): Promise<void>;
+    abort(reason: string): void;
+}
 export interface ImageDescriptionRequest {
     buffer: Buffer;
     fileName: string;
@@ -170,6 +229,8 @@ export interface MediaUnderstandingProvider {
     /** True when the provider can natively ingest PDF (no OCR pre-step needed). */
     nativeDocumentInputs?: ReadonlyArray<'pdf'>;
     transcribeAudio?: (req: AudioTranscriptionRequest) => Promise<AudioTranscriptionResult>;
+    streamingAudio?: StreamingSttCapabilities;
+    openAudioStream?: (req: StreamingSttOpenRequest) => Promise<StreamingSttSession>;
     describeImage?: (req: ImageDescriptionRequest) => Promise<ImageDescriptionResult>;
     describeVideo?: (req: VideoDescriptionRequest) => Promise<VideoDescriptionResult>;
 }

package/dist/src/providers/auth-runtime/resolve-auth.js

@@ -1,5 +1,5 @@
-import { getApiKeyFromEnv, init_env_keys } from "../env-keys.js";
 import { init_sync_provider_auth, resolveProviderApiKeyForAgentSync } from "../../auth/sync-provider-auth.js";
+import { getApiKeyFromEnv, init_env_keys } from "../env-keys.js";
 import { getDefaultAuthProfileStore } from "./auth-profile-store.js";
 //#region src/providers/auth-runtime/resolve-auth.ts
 init_env_keys();

package/dist/src/providers/index.js

@@ -1,11 +1,11 @@
 import { __esmMin, __exportAll } from "../../_virtual/_rolldown/runtime.js";
 import { getAgentDefaultModelRef, init_schema } from "../config/schema.js";
+import { getModelOAuthProviderIds, init_registry } from "../auth/oauth/registry.js";
+import { hasProviderAuthOnDiskSync, init_sync_provider_auth } from "../auth/sync-provider-auth.js";
 import { PROVIDER_ENV_MAP, getApiKeyFromEnv, init_env_keys } from "./env-keys.js";
 import { init_model_ref, splitProviderModelRef } from "./model-ref.js";
 import { ModelRegistry, getModelRegistry, init_model_registry, prewarmModelRegistry, resetModelRegistry } from "./model-registry.js";
-import { getModelOAuthProviderIds, init_registry } from "../auth/oauth/registry.js";
 import { CredentialResolver, hasCredentials, init_credentials } from "../auth/credentials.js";
-import { hasProviderAuthOnDiskSync, init_sync_provider_auth } from "../auth/sync-provider-auth.js";
 import { EXTENSION_PROVIDER_BASE_URL, init_constants } from "./constants.js";
 import { applyOfficialModelMetadataCorrections, getSupplementalModels, init_model_supplements } from "./model-supplements.js";
 import { getProviderRegistry, init_plugin_registry } from "./plugin-registry.js";

package/dist/src/providers/model-registry.js

@@ -1,8 +1,8 @@
 import { __esmMin } from "../../_virtual/_rolldown/runtime.js";
 import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
-import { init_resolve_config_value, resolveConfigValue, resolveHeaders } from "../config/resolve-config-value.js";
 import { resolveModelsJsonPath } from "../config/paths.js";
+import { init_resolve_config_value, resolveConfigValue, resolveHeaders } from "../config/resolve-config-value.js";
 import { getDefaultModelValues, init_models_json, validateModelsConfig } from "../config/models-json.js";
 import { getApiKeyFromEnv, init_env_keys } from "./env-keys.js";
 import { getModelCatalogStore, init_model_catalog_store } from "./model-catalog-store.js";