@openclaw/voice-call 2026.5.2 → 2026.5.3-beta.1

package/dist/realtime-handler-B63CIDP2.js

@@ -0,0 +1,325 @@
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, buildRealtimeVoiceAgentConsultWorkingResponse, createRealtimeVoiceBridgeSession } from "openclaw/plugin-sdk/realtime-voice";
+import { randomUUID } from "node:crypto";
+import WebSocket, { WebSocketServer } from "ws";
+//#region extensions/voice-call/src/webhook/realtime-handler.ts
+const STREAM_TOKEN_TTL_MS = 3e4;
+const DEFAULT_HOST = "localhost:8443";
+const MAX_REALTIME_MESSAGE_BYTES = 256 * 1024;
+function normalizePath(pathname) {
+	const trimmed = pathname.trim();
+	if (!trimmed) return "/";
+	const prefixed = trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+	if (prefixed === "/") return prefixed;
+	return prefixed.endsWith("/") ? prefixed.slice(0, -1) : prefixed;
+}
+function buildGreetingInstructions(baseInstructions, greeting) {
+	const trimmedGreeting = greeting?.trim();
+	if (!trimmedGreeting) return;
+	const intro = "Start the call by greeting the caller naturally. Include this greeting in your first spoken reply:";
+	return baseInstructions ? `${baseInstructions}\n\n${intro} "${trimmedGreeting}"` : `${intro} "${trimmedGreeting}"`;
+}
+var RealtimeCallHandler = class {
+	constructor(config, manager, provider, realtimeProvider, providerConfig, servePath) {
+		this.config = config;
+		this.manager = manager;
+		this.provider = provider;
+		this.realtimeProvider = realtimeProvider;
+		this.providerConfig = providerConfig;
+		this.servePath = servePath;
+		this.toolHandlers = /* @__PURE__ */ new Map();
+		this.pendingStreamTokens = /* @__PURE__ */ new Map();
+		this.activeBridgesByCallId = /* @__PURE__ */ new Map();
+		this.partialUserTranscriptsByCallId = /* @__PURE__ */ new Map();
+		this.publicOrigin = null;
+		this.publicPathPrefix = "";
+	}
+	setPublicUrl(url) {
+		try {
+			const parsed = new URL(url);
+			this.publicOrigin = parsed.host;
+			const normalizedServePath = normalizePath(this.servePath);
+			const normalizedPublicPath = normalizePath(parsed.pathname);
+			const idx = normalizedPublicPath.indexOf(normalizedServePath);
+			this.publicPathPrefix = idx > 0 ? normalizedPublicPath.slice(0, idx) : "";
+		} catch {
+			this.publicOrigin = null;
+			this.publicPathPrefix = "";
+		}
+	}
+	getStreamPathPattern() {
+		return `${this.publicPathPrefix}${normalizePath(this.config.streamPath ?? "/voice/stream/realtime")}`;
+	}
+	buildTwiMLPayload(req, params) {
+		const host = this.publicOrigin || req.headers.host || DEFAULT_HOST;
+		const rawDirection = params?.get("Direction");
+		const token = this.issueStreamToken({
+			from: params?.get("From") ?? void 0,
+			to: params?.get("To") ?? void 0,
+			direction: rawDirection?.startsWith("outbound") ? "outbound" : "inbound"
+		});
+		return {
+			statusCode: 200,
+			headers: { "Content-Type": "text/xml" },
+			body: `<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <Connect>
+    <Stream url="${`wss://${host}${this.getStreamPathPattern()}/${token}`}" />
+  </Connect>
+</Response>`
+		};
+	}
+	handleWebSocketUpgrade(request, socket, head) {
+		const token = new URL(request.url ?? "/", "wss://localhost").pathname.split("/").pop() ?? null;
+		const callerMeta = token ? this.consumeStreamToken(token) : null;
+		if (!callerMeta) {
+			socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
+			socket.destroy();
+			return;
+		}
+		new WebSocketServer({
+			noServer: true,
+			maxPayload: MAX_REALTIME_MESSAGE_BYTES
+		}).handleUpgrade(request, socket, head, (ws) => {
+			let bridge = null;
+			let initialized = false;
+			ws.on("message", (data) => {
+				try {
+					const msg = JSON.parse(data.toString());
+					if (!initialized && msg.event === "start") {
+						initialized = true;
+						const startData = typeof msg.start === "object" && msg.start !== null ? msg.start : void 0;
+						const streamSid = typeof startData?.streamSid === "string" ? startData.streamSid : "unknown";
+						const callSid = typeof startData?.callSid === "string" ? startData.callSid : "unknown";
+						const nextBridge = this.handleCall(streamSid, callSid, ws, callerMeta);
+						if (!nextBridge) return;
+						bridge = nextBridge;
+						return;
+					}
+					if (!bridge) return;
+					const mediaData = typeof msg.media === "object" && msg.media !== null ? msg.media : void 0;
+					if (msg.event === "media" && typeof mediaData?.payload === "string") {
+						bridge.sendAudio(Buffer.from(mediaData.payload, "base64"));
+						if (typeof mediaData.timestamp === "number") bridge.setMediaTimestamp(mediaData.timestamp);
+						else if (typeof mediaData.timestamp === "string") bridge.setMediaTimestamp(Number.parseInt(mediaData.timestamp, 10));
+						return;
+					}
+					if (msg.event === "mark") {
+						bridge.acknowledgeMark();
+						return;
+					}
+					if (msg.event === "stop") bridge.close();
+				} catch (error) {
+					console.error("[voice-call] realtime WS parse failed:", error);
+				}
+			});
+			ws.on("close", () => {
+				bridge?.close();
+			});
+			ws.on("error", (error) => {
+				console.error("[voice-call] realtime WS error:", error);
+			});
+		});
+	}
+	registerToolHandler(name, fn) {
+		this.toolHandlers.set(name, fn);
+	}
+	speak(callId, instructions) {
+		const bridge = this.activeBridgesByCallId.get(callId);
+		if (!bridge) return {
+			success: false,
+			error: "No active realtime bridge for call"
+		};
+		try {
+			bridge.triggerGreeting(instructions);
+			return { success: true };
+		} catch (error) {
+			return {
+				success: false,
+				error: formatErrorMessage(error)
+			};
+		}
+	}
+	issueStreamToken(meta = {}) {
+		const token = randomUUID();
+		this.pendingStreamTokens.set(token, {
+			expiry: Date.now() + STREAM_TOKEN_TTL_MS,
+			...meta
+		});
+		for (const [candidate, entry] of this.pendingStreamTokens) if (Date.now() > entry.expiry) this.pendingStreamTokens.delete(candidate);
+		return token;
+	}
+	consumeStreamToken(token) {
+		const entry = this.pendingStreamTokens.get(token);
+		if (!entry) return null;
+		this.pendingStreamTokens.delete(token);
+		if (Date.now() > entry.expiry) return null;
+		return {
+			from: entry.from,
+			to: entry.to,
+			direction: entry.direction
+		};
+	}
+	handleCall(streamSid, callSid, ws, callerMeta) {
+		const registration = this.registerCallInManager(callSid, callerMeta);
+		if (!registration) {
+			ws.close(1008, "Caller rejected by policy");
+			return null;
+		}
+		const { callId, initialGreetingInstructions } = registration;
+		console.log(`[voice-call] Realtime bridge starting for call ${callId} (providerCallId=${callSid}, initialGreeting=${initialGreetingInstructions ? "queued" : "absent"})`);
+		let callEndEmitted = false;
+		const emitCallEnd = (reason) => {
+			if (callEndEmitted) return;
+			callEndEmitted = true;
+			this.endCallInManager(callSid, callId, reason);
+		};
+		const bridge = createRealtimeVoiceBridgeSession({
+			provider: this.realtimeProvider,
+			providerConfig: this.providerConfig,
+			instructions: this.config.instructions,
+			tools: this.config.tools,
+			initialGreetingInstructions,
+			triggerGreetingOnReady: Boolean(initialGreetingInstructions),
+			audioSink: {
+				isOpen: () => ws.readyState === WebSocket.OPEN,
+				sendAudio: (muLaw) => {
+					ws.send(JSON.stringify({
+						event: "media",
+						streamSid,
+						media: { payload: muLaw.toString("base64") }
+					}));
+				},
+				clearAudio: () => {
+					ws.send(JSON.stringify({
+						event: "clear",
+						streamSid
+					}));
+				},
+				sendMark: (markName) => {
+					ws.send(JSON.stringify({
+						event: "mark",
+						streamSid,
+						mark: { name: markName }
+					}));
+				}
+			},
+			onTranscript: (role, text, isFinal) => {
+				if (!isFinal) {
+					if (role === "user" && text.trim()) this.partialUserTranscriptsByCallId.set(callId, text);
+					return;
+				}
+				if (role === "user") {
+					this.partialUserTranscriptsByCallId.delete(callId);
+					const event = {
+						id: `realtime-speech-${callSid}-${Date.now()}`,
+						type: "call.speech",
+						callId,
+						providerCallId: callSid,
+						timestamp: Date.now(),
+						transcript: text,
+						isFinal: true
+					};
+					this.manager.processEvent(event);
+					return;
+				}
+				this.manager.processEvent({
+					id: `realtime-bot-${callSid}-${Date.now()}`,
+					type: "call.speaking",
+					callId,
+					providerCallId: callSid,
+					timestamp: Date.now(),
+					text
+				});
+			},
+			onToolCall: (toolEvent, session) => {
+				this.executeToolCall(session, callId, toolEvent.callId || toolEvent.itemId, toolEvent.name, toolEvent.args);
+			},
+			onError: (error) => {
+				console.error("[voice-call] realtime voice error:", error.message);
+			},
+			onClose: (reason) => {
+				this.activeBridgesByCallId.delete(callId);
+				this.activeBridgesByCallId.delete(callSid);
+				this.partialUserTranscriptsByCallId.delete(callId);
+				if (reason !== "error") return;
+				emitCallEnd("error");
+				if (ws.readyState === WebSocket.OPEN) ws.close(1011, "Bridge disconnected");
+				this.provider.hangupCall({
+					callId,
+					providerCallId: callSid,
+					reason: "error"
+				}).catch((error) => {
+					console.warn(`[voice-call] Failed to hang up realtime call ${callSid}: ${formatErrorMessage(error)}`);
+				});
+			}
+		});
+		this.activeBridgesByCallId.set(callId, bridge);
+		this.activeBridgesByCallId.set(callSid, bridge);
+		const closeBridge = bridge.close.bind(bridge);
+		bridge.close = () => {
+			this.activeBridgesByCallId.delete(callId);
+			this.activeBridgesByCallId.delete(callSid);
+			this.partialUserTranscriptsByCallId.delete(callId);
+			closeBridge();
+		};
+		bridge.connect().catch((error) => {
+			console.error("[voice-call] Failed to connect realtime bridge:", error);
+			bridge.close();
+			emitCallEnd("error");
+			ws.close(1011, "Failed to connect");
+		});
+		return bridge;
+	}
+	registerCallInManager(callSid, callerMeta = {}) {
+		const baseFields = {
+			providerCallId: callSid,
+			timestamp: Date.now(),
+			direction: callerMeta.direction ?? "inbound",
+			...callerMeta.from ? { from: callerMeta.from } : {},
+			...callerMeta.to ? { to: callerMeta.to } : {}
+		};
+		this.manager.processEvent({
+			id: `realtime-initiated-${callSid}`,
+			callId: callSid,
+			type: "call.initiated",
+			...baseFields
+		});
+		const callRecord = this.manager.getCallByProviderCallId(callSid);
+		if (!callRecord) return null;
+		const initialGreeting = this.extractInitialGreeting(callRecord);
+		console.log(`[voice-call] Realtime call ${callRecord.callId} initial greeting ${initialGreeting ? "queued" : "absent"}`);
+		if (callRecord.metadata) delete callRecord.metadata.initialMessage;
+		this.manager.processEvent({
+			id: `realtime-answered-${callSid}`,
+			callId: callSid,
+			type: "call.answered",
+			...baseFields
+		});
+		return {
+			callId: callRecord.callId,
+			initialGreetingInstructions: buildGreetingInstructions(this.config.instructions, initialGreeting)
+		};
+	}
+	extractInitialGreeting(call) {
+		return typeof call.metadata?.initialMessage === "string" ? call.metadata.initialMessage : void 0;
+	}
+	endCallInManager(callSid, callId, reason) {
+		this.manager.processEvent({
+			id: `realtime-ended-${callSid}-${Date.now()}`,
+			type: "call.ended",
+			callId,
+			providerCallId: callSid,
+			timestamp: Date.now(),
+			reason
+		});
+	}
+	async executeToolCall(bridge, callId, bridgeCallId, name, args) {
+		const handler = this.toolHandlers.get(name);
+		if (handler && name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME && bridge.bridge.supportsToolResultContinuation && !this.config.fastContext.enabled) bridge.submitToolResult(bridgeCallId, buildRealtimeVoiceAgentConsultWorkingResponse("caller"), { willContinue: true });
+		const result = !handler ? { error: `Tool "${name}" not available` } : await handler(args, callId, { partialUserTranscript: this.partialUserTranscriptsByCallId.get(callId) }).catch((error) => ({ error: formatErrorMessage(error) }));
+		bridge.submitToolResult(bridgeCallId, result);
+	}
+};
+//#endregion
+export { RealtimeCallHandler };

package/dist/realtime-transcription.runtime-B2h70y2W.js

@@ -0,0 +1,2 @@
+import { getRealtimeTranscriptionProvider, listRealtimeTranscriptionProviders } from "openclaw/plugin-sdk/realtime-transcription";
+export { getRealtimeTranscriptionProvider, listRealtimeTranscriptionProviders };

package/dist/realtime-voice.runtime-Bkh4nvLn.js

@@ -0,0 +1,2 @@
+import { getRealtimeVoiceProvider, listRealtimeVoiceProviders, resolveConfiguredRealtimeVoiceProvider } from "openclaw/plugin-sdk/realtime-voice";
+export { getRealtimeVoiceProvider, listRealtimeVoiceProviders, resolveConfiguredRealtimeVoiceProvider };

package/dist/response-generator-BrcmwDZU.js

@@ -0,0 +1,182 @@
+import { o as resolveVoiceCallSessionKey } from "./config-7w04YpHh.js";
+import { t as resolveVoiceResponseModel } from "./response-model-CyF5K80p.js";
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
+import crypto from "node:crypto";
+import { applyModelOverrideToSessionEntry } from "openclaw/plugin-sdk/model-session-runtime";
+//#region extensions/voice-call/src/response-generator.ts
+/**
+* Voice call response generator - uses the embedded Pi agent for tool support.
+* Routes voice responses through the same agent infrastructure as messaging.
+*/
+const VOICE_SPOKEN_OUTPUT_CONTRACT = [
+	"Output format requirements:",
+	"- Return only valid JSON in this exact shape: {\"spoken\":\"...\"}",
+	"- Do not include markdown, code fences, planning text, or extra keys.",
+	"- Put exactly what should be spoken to the caller into \"spoken\".",
+	"- If there is nothing to say, return {\"spoken\":\"\"}."
+].join("\n");
+function normalizeSpokenText(value) {
+	const normalized = value.replace(/\s+/g, " ").trim();
+	return normalized.length > 0 ? normalized : null;
+}
+function tryParseSpokenJson(text) {
+	const candidates = [];
+	const trimmed = text.trim();
+	if (!trimmed) return null;
+	candidates.push(trimmed);
+	const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
+	if (fenced?.[1]) candidates.push(fenced[1]);
+	const firstBrace = trimmed.indexOf("{");
+	const lastBrace = trimmed.lastIndexOf("}");
+	if (firstBrace >= 0 && lastBrace > firstBrace) candidates.push(trimmed.slice(firstBrace, lastBrace + 1));
+	for (const candidate of candidates) try {
+		const parsed = JSON.parse(candidate);
+		if (typeof parsed?.spoken !== "string") continue;
+		return normalizeSpokenText(parsed.spoken) ?? "";
+	} catch {}
+	const inlineSpokenMatch = trimmed.match(/"spoken"\s*:\s*"((?:[^"\\]|\\.)*)"/i);
+	if (!inlineSpokenMatch) return null;
+	try {
+		return normalizeSpokenText(JSON.parse(`"${inlineSpokenMatch[1] ?? ""}"`)) ?? "";
+	} catch {
+		return null;
+	}
+}
+function isLikelyMetaReasoningParagraph(paragraph) {
+	const lower = normalizeLowercaseStringOrEmpty(paragraph);
+	if (!lower) return false;
+	if (lower.startsWith("thinking process")) return true;
+	if (lower.startsWith("reasoning:") || lower.startsWith("analysis:")) return true;
+	if (lower.startsWith("the user ") && (lower.includes("i should") || lower.includes("i need to") || lower.includes("i will"))) return true;
+	if (lower.includes("this is a natural continuation of the conversation") || lower.includes("keep the conversation flowing")) return true;
+	return false;
+}
+function sanitizePlainSpokenText(text) {
+	const withoutCodeFences = text.replace(/```[\s\S]*?```/g, " ").trim();
+	if (!withoutCodeFences) return null;
+	const paragraphs = withoutCodeFences.split(/\n\s*\n+/).map((paragraph) => paragraph.trim()).filter(Boolean);
+	while (paragraphs.length > 1 && isLikelyMetaReasoningParagraph(paragraphs[0])) paragraphs.shift();
+	return normalizeSpokenText(paragraphs.join(" "));
+}
+function extractSpokenTextFromPayloads(payloads) {
+	const spokenSegments = [];
+	for (const payload of payloads) {
+		if (payload.isError || payload.isReasoning) continue;
+		const rawText = payload.text?.trim() ?? "";
+		if (!rawText) continue;
+		const structured = tryParseSpokenJson(rawText);
+		if (structured !== null) {
+			if (structured.length > 0) spokenSegments.push(structured);
+			continue;
+		}
+		const plain = sanitizePlainSpokenText(rawText);
+		if (plain) spokenSegments.push(plain);
+	}
+	return spokenSegments.length > 0 ? spokenSegments.join(" ").trim() : null;
+}
+function resolveVoiceSandboxSessionKey(agentId, sessionKey) {
+	const trimmed = sessionKey.trim();
+	if (trimmed.toLowerCase().startsWith("agent:")) return trimmed;
+	return `agent:${agentId}:${trimmed}`;
+}
+/**
+* Generate a voice response using the embedded Pi agent with full tool support.
+* Uses the same agent infrastructure as messaging for consistent behavior.
+*/
+async function generateVoiceResponse(params) {
+	const { voiceConfig, callId, sessionKey, from, transcript, userMessage, coreConfig, agentRuntime } = params;
+	if (!coreConfig) return {
+		text: null,
+		error: "Core config unavailable for voice response"
+	};
+	const cfg = coreConfig;
+	const resolvedSessionKey = resolveVoiceCallSessionKey({
+		config: voiceConfig,
+		callId,
+		phone: from,
+		explicitSessionKey: sessionKey
+	});
+	const agentId = voiceConfig.agentId ?? "main";
+	const storePath = agentRuntime.session.resolveStorePath(cfg.session?.store, { agentId });
+	const agentDir = agentRuntime.resolveAgentDir(cfg, agentId);
+	const workspaceDir = agentRuntime.resolveAgentWorkspaceDir(cfg, agentId);
+	await agentRuntime.ensureAgentWorkspace({ dir: workspaceDir });
+	const sessionStore = agentRuntime.session.loadSessionStore(storePath);
+	const now = Date.now();
+	const existingSessionEntry = sessionStore[resolvedSessionKey];
+	const { provider, model } = resolveVoiceResponseModel({
+		voiceConfig,
+		agentRuntime
+	});
+	let sessionEntry = existingSessionEntry;
+	if (!sessionEntry?.sessionId || voiceConfig.responseModel) sessionEntry = await agentRuntime.session.updateSessionStore(storePath, (store) => {
+		let entry = store[resolvedSessionKey];
+		if (!entry?.sessionId) {
+			entry = {
+				...entry,
+				sessionId: crypto.randomUUID(),
+				updatedAt: now
+			};
+			store[resolvedSessionKey] = entry;
+		}
+		if (voiceConfig.responseModel) applyModelOverrideToSessionEntry({
+			entry,
+			selection: {
+				provider,
+				model
+			},
+			selectionSource: "auto"
+		});
+		return entry;
+	});
+	const sessionId = sessionEntry.sessionId;
+	const sessionFile = agentRuntime.session.resolveSessionFilePath(sessionId, sessionEntry, { agentId });
+	const thinkLevel = agentRuntime.resolveThinkingDefault({
+		cfg,
+		provider,
+		model
+	});
+	const agentName = agentRuntime.resolveAgentIdentity(cfg, agentId)?.name?.trim() || "assistant";
+	const basePrompt = voiceConfig.responseSystemPrompt ?? `You are ${agentName}, a helpful voice assistant on a phone call. Keep responses brief and conversational (1-2 sentences max). Be natural and friendly. The caller's phone number is ${from}. You have access to tools - use them when helpful.`;
+	let extraSystemPrompt = basePrompt;
+	if (transcript.length > 0) extraSystemPrompt = `${basePrompt}\n\nConversation so far:\n${transcript.map((entry) => `${entry.speaker === "bot" ? "You" : "Caller"}: ${entry.text}`).join("\n")}`;
+	extraSystemPrompt = `${extraSystemPrompt}\n\n${VOICE_SPOKEN_OUTPUT_CONTRACT}`;
+	const timeoutMs = voiceConfig.responseTimeoutMs ?? agentRuntime.resolveAgentTimeoutMs({ cfg });
+	const runId = `voice:${callId}:${Date.now()}`;
+	try {
+		const result = await agentRuntime.runEmbeddedPiAgent({
+			sessionId,
+			sessionKey: resolvedSessionKey,
+			sandboxSessionKey: resolveVoiceSandboxSessionKey(agentId, resolvedSessionKey),
+			agentId,
+			messageProvider: "voice",
+			sessionFile,
+			workspaceDir,
+			config: cfg,
+			prompt: userMessage,
+			provider,
+			model,
+			thinkLevel,
+			verboseLevel: "off",
+			timeoutMs,
+			runId,
+			lane: "voice",
+			extraSystemPrompt,
+			agentDir
+		});
+		const text = extractSpokenTextFromPayloads(result.payloads ?? []);
+		if (!text && result.meta?.aborted) return {
+			text: null,
+			error: "Response generation was aborted"
+		};
+		return { text };
+	} catch (err) {
+		console.error(`[voice-call] Response generation failed:`, err);
+		return {
+			text: null,
+			error: String(err)
+		};
+	}
+}
+//#endregion
+export { generateVoiceResponse };

package/dist/response-model-CyF5K80p.js

@@ -0,0 +1,12 @@
+//#region extensions/voice-call/src/response-model.ts
+function resolveVoiceResponseModel(params) {
+	const modelRef = params.voiceConfig.responseModel ?? `${params.agentRuntime.defaults.provider}/${params.agentRuntime.defaults.model}`;
+	const slashIndex = modelRef.indexOf("/");
+	return {
+		modelRef,
+		provider: slashIndex === -1 ? params.agentRuntime.defaults.provider : modelRef.slice(0, slashIndex),
+		model: slashIndex === -1 ? modelRef : modelRef.slice(slashIndex + 1)
+	};
+}
+//#endregion
+export { resolveVoiceResponseModel as t };

package/dist/runtime-api.js

@@ -0,0 +1,6 @@
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+import { isRequestBodyLimitError, readRequestBodyWithLimit, requestBodyErrorToText } from "openclaw/plugin-sdk/webhook-request-guards";
+import { fetchWithSsrFGuard, isBlockedHostnameOrIp } from "openclaw/plugin-sdk/ssrf-runtime";
+import { TtsAutoSchema, TtsConfigSchema, TtsModeSchema, TtsProviderSchema } from "openclaw/plugin-sdk/tts-runtime";
+import { sleep } from "openclaw/plugin-sdk/runtime-env";
+export { TtsAutoSchema, TtsConfigSchema, TtsModeSchema, TtsProviderSchema, definePluginEntry, fetchWithSsrFGuard, isBlockedHostnameOrIp, isRequestBodyLimitError, readRequestBodyWithLimit, requestBodyErrorToText, sleep };