@openclaw/voice-call 2026.3.13 → 2026.5.2-beta.1

package/src/runtime.ts

@@ -1,16 +1,36 @@
+import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import {
+  consultRealtimeVoiceAgent,
+  REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
+  resolveRealtimeVoiceAgentConsultTools,
+  resolveRealtimeVoiceAgentConsultToolsAllow,
+  type RealtimeVoiceAgentConsultTranscriptEntry,
+  type ResolvedRealtimeVoiceProvider,
+} from "openclaw/plugin-sdk/realtime-voice";
 import type { VoiceCallConfig } from "./config.js";
-import { resolveVoiceCallConfig, validateProviderConfig } from "./config.js";
-import type { CoreConfig } from "./core-bridge.js";
+import {
+  resolveVoiceCallEffectiveConfig,
+  resolveVoiceCallSessionKey,
+  resolveTwilioAuthToken,
+  resolveVoiceCallConfig,
+  validateProviderConfig,
+} from "./config.js";
+import type { CoreAgentDeps, CoreConfig } from "./core-bridge.js";
 import { CallManager } from "./manager.js";
 import type { VoiceCallProvider } from "./providers/base.js";
-import { MockProvider } from "./providers/mock.js";
-import { PlivoProvider } from "./providers/plivo.js";
-import { TelnyxProvider } from "./providers/telnyx.js";
-import { TwilioProvider } from "./providers/twilio.js";
+import type { TwilioProvider } from "./providers/twilio.js";
+import { resolveRealtimeFastContextConsult } from "./realtime-fast-context.js";
+import { resolveVoiceResponseModel } from "./response-model.js";
 import type { TelephonyTtsRuntime } from "./telephony-tts.js";
 import { createTelephonyTtsProvider } from "./telephony-tts.js";
 import { startTunnel, type TunnelResult } from "./tunnel.js";
+import {
+  isProviderUnreachableWebhookUrl,
+  providerRequiresPublicWebhook,
+} from "./webhook-exposure.js";
 import { VoiceCallWebhookServer } from "./webhook.js";
+import type { ToolHandlerContext } from "./webhook/realtime-handler.js";
 import { cleanupTailscaleExposure, setupTailscaleExposure } from "./webhook/tailscale.js";
 
 export type VoiceCallRuntime = {
@@ -30,6 +50,98 @@ type Logger = {
   debug?: (message: string) => void;
 };
 
+type ResolvedRealtimeProvider = ResolvedRealtimeVoiceProvider;
+
+type TelnyxProviderModule = typeof import("./providers/telnyx.js");
+type TwilioProviderModule = typeof import("./providers/twilio.js");
+type PlivoProviderModule = typeof import("./providers/plivo.js");
+type MockProviderModule = typeof import("./providers/mock.js");
+type RealtimeVoiceRuntimeModule = typeof import("./realtime-voice.runtime.js");
+type RealtimeHandlerModule = typeof import("./webhook/realtime-handler.js");
+
+const REALTIME_VOICE_CONSULT_SYSTEM_PROMPT = [
+  "You are a behind-the-scenes consultant for a live phone voice agent.",
+  "Prioritize a fast, speakable answer over exhaustive investigation.",
+  "For tool-backed status checks, prefer one or two bounded read-only queries before answering.",
+  "Do not print secret values or dump environment variables; only check whether required configuration is present.",
+  "Be accurate, brief, and speakable.",
+].join(" ");
+
+let telnyxProviderPromise: Promise<TelnyxProviderModule> | undefined;
+let twilioProviderPromise: Promise<TwilioProviderModule> | undefined;
+let plivoProviderPromise: Promise<PlivoProviderModule> | undefined;
+let mockProviderPromise: Promise<MockProviderModule> | undefined;
+let realtimeVoiceRuntimePromise: Promise<RealtimeVoiceRuntimeModule> | undefined;
+let realtimeHandlerPromise: Promise<RealtimeHandlerModule> | undefined;
+
+function loadTelnyxProvider(): Promise<TelnyxProviderModule> {
+  telnyxProviderPromise ??= import("./providers/telnyx.js");
+  return telnyxProviderPromise;
+}
+
+function loadTwilioProvider(): Promise<TwilioProviderModule> {
+  twilioProviderPromise ??= import("./providers/twilio.js");
+  return twilioProviderPromise;
+}
+
+function loadPlivoProvider(): Promise<PlivoProviderModule> {
+  plivoProviderPromise ??= import("./providers/plivo.js");
+  return plivoProviderPromise;
+}
+
+function loadMockProvider(): Promise<MockProviderModule> {
+  mockProviderPromise ??= import("./providers/mock.js");
+  return mockProviderPromise;
+}
+
+function loadRealtimeVoiceRuntime(): Promise<RealtimeVoiceRuntimeModule> {
+  realtimeVoiceRuntimePromise ??= import("./realtime-voice.runtime.js");
+  return realtimeVoiceRuntimePromise;
+}
+
+function loadRealtimeHandler(): Promise<RealtimeHandlerModule> {
+  realtimeHandlerPromise ??= import("./webhook/realtime-handler.js");
+  return realtimeHandlerPromise;
+}
+
+function resolveVoiceCallConsultSessionKey(call: {
+  config: VoiceCallConfig;
+  sessionKey?: string;
+  from?: string;
+  to?: string;
+  direction?: "inbound" | "outbound";
+  callId: string;
+}): string {
+  if (call.sessionKey) {
+    return call.sessionKey;
+  }
+  const phone = call.direction === "outbound" ? call.to : call.from;
+  return resolveVoiceCallSessionKey({
+    config: call.config,
+    callId: call.callId,
+    phone,
+  });
+}
+
+function mapVoiceCallConsultTranscript(
+  call: {
+    transcript?: Array<{ speaker: "user" | "bot"; text: string }>;
+  },
+  context?: ToolHandlerContext,
+): RealtimeVoiceAgentConsultTranscriptEntry[] {
+  const transcript: RealtimeVoiceAgentConsultTranscriptEntry[] = (call.transcript ?? []).map(
+    (entry) => ({
+      role: entry.speaker === "bot" ? "assistant" : "user",
+      text: entry.text,
+    }),
+  );
+  const partial = context?.partialUserTranscript?.trim();
+  if (partial && transcript.at(-1)?.text !== partial) {
+    transcript.push({ role: "user", text: partial });
+  }
+  return transcript;
+}
+
 function createRuntimeResourceLifecycle(params: {
   config: VoiceCallConfig;
   webhookServer: VoiceCallWebhookServer;
@@ -80,14 +192,15 @@ function isLoopbackBind(bind: string | undefined): boolean {
   return bind === "127.0.0.1" || bind === "::1" || bind === "localhost";
 }
 
-function resolveProvider(config: VoiceCallConfig): VoiceCallProvider {
+async function resolveProvider(config: VoiceCallConfig): Promise<VoiceCallProvider> {
   const allowNgrokFreeTierLoopbackBypass =
     config.tunnel?.provider === "ngrok" &&
     isLoopbackBind(config.serve?.bind) &&
     (config.tunnel?.allowNgrokFreeTierLoopbackBypass ?? false);
 
   switch (config.provider) {
-    case "telnyx":
+    case "telnyx": {
+      const { TelnyxProvider } = await loadTelnyxProvider();
       return new TelnyxProvider(
         {
           apiKey: config.telnyx?.apiKey,
@@ -98,11 +211,13 @@ function resolveProvider(config: VoiceCallConfig): VoiceCallProvider {
           skipVerification: config.skipSignatureVerification,
         },
       );
-    case "twilio":
+    }
+    case "twilio": {
+      const { TwilioProvider } = await loadTwilioProvider();
       return new TwilioProvider(
         {
           accountSid: config.twilio?.accountSid,
-          authToken: config.twilio?.authToken,
+          authToken: resolveTwilioAuthToken(config),
         },
         {
           allowNgrokFreeTierLoopbackBypass,
@@ -112,7 +227,9 @@ function resolveProvider(config: VoiceCallConfig): VoiceCallProvider {
           webhookSecurity: config.webhookSecurity,
         },
       );
-    case "plivo":
+    }
+    case "plivo": {
+      const { PlivoProvider } = await loadPlivoProvider();
       return new PlivoProvider(
         {
           authId: config.plivo?.authId,
@@ -125,20 +242,37 @@ function resolveProvider(config: VoiceCallConfig): VoiceCallProvider {
           webhookSecurity: config.webhookSecurity,
         },
       );
-    case "mock":
+    }
+    case "mock": {
+      const { MockProvider } = await loadMockProvider();
       return new MockProvider();
+    }
     default:
       throw new Error(`Unsupported voice-call provider: ${String(config.provider)}`);
   }
 }
 
+async function resolveRealtimeProvider(params: {
+  config: VoiceCallConfig;
+  fullConfig: OpenClawConfig;
+}): Promise<ResolvedRealtimeProvider> {
+  const { resolveConfiguredRealtimeVoiceProvider } = await loadRealtimeVoiceRuntime();
+  return resolveConfiguredRealtimeVoiceProvider({
+    configuredProviderId: params.config.realtime.provider,
+    providerConfigs: params.config.realtime.providers,
+    cfg: params.fullConfig,
+  });
+}
+
 export async function createVoiceCallRuntime(params: {
   config: VoiceCallConfig;
   coreConfig: CoreConfig;
+  fullConfig?: OpenClawConfig;
+  agentRuntime: CoreAgentDeps;
   ttsRuntime?: TelephonyTtsRuntime;
   logger?: Logger;
 }): Promise<VoiceCallRuntime> {
-  const { config: rawConfig, coreConfig, ttsRuntime, logger } = params;
+  const { config: rawConfig, coreConfig, fullConfig, agentRuntime, ttsRuntime, logger } = params;
   const log = logger ?? {
     info: console.log,
     warn: console.warn,
@@ -147,6 +281,7 @@ export async function createVoiceCallRuntime(params: {
   };
 
   const config = resolveVoiceCallConfig(rawConfig);
+  const cfg = fullConfig ?? (coreConfig as OpenClawConfig);
 
   if (!config.enabled) {
     throw new Error("Voice call disabled. Enable the plugin entry in config.");
@@ -163,9 +298,107 @@ export async function createVoiceCallRuntime(params: {
     throw new Error(`Invalid voice-call config: ${validation.errors.join("; ")}`);
   }
 
-  const provider = resolveProvider(config);
+  const provider = await resolveProvider(config);
   const manager = new CallManager(config);
-  const webhookServer = new VoiceCallWebhookServer(config, manager, provider, coreConfig);
+  const realtimeProvider = config.realtime.enabled
+    ? await resolveRealtimeProvider({
+        config,
+        fullConfig: cfg,
+      })
+    : null;
+  const webhookServer = new VoiceCallWebhookServer(
+    config,
+    manager,
+    provider,
+    coreConfig,
+    fullConfig ?? (coreConfig as OpenClawConfig),
+    agentRuntime,
+    log,
+  );
+  if (realtimeProvider) {
+    const { RealtimeCallHandler } = await loadRealtimeHandler();
+    const realtimeConfig = {
+      ...config.realtime,
+      tools: resolveRealtimeVoiceAgentConsultTools(
+        config.realtime.toolPolicy,
+        config.realtime.tools,
+      ),
+    };
+    const realtimeHandler = new RealtimeCallHandler(
+      realtimeConfig,
+      manager,
+      provider,
+      realtimeProvider.provider,
+      realtimeProvider.providerConfig,
+      config.serve.path,
+    );
+    if (config.realtime.toolPolicy !== "none") {
+      realtimeHandler.registerToolHandler(
+        REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
+        async (args, callId, handlerContext) => {
+          const call = manager.getCall(callId);
+          if (!call) {
+            return { error: `Call "${callId}" not found` };
+          }
+          const numberRouteKey =
+            typeof call.metadata?.numberRouteKey === "string"
+              ? call.metadata.numberRouteKey
+              : call.to;
+          const effectiveConfig = resolveVoiceCallEffectiveConfig(config, numberRouteKey).config;
+          const agentId = effectiveConfig.agentId ?? "main";
+          const sessionKey = resolveVoiceCallConsultSessionKey({
+            ...call,
+            config: effectiveConfig,
+          });
+          const fastContext = await resolveRealtimeFastContextConsult({
+            cfg,
+            agentId,
+            sessionKey,
+            config: effectiveConfig.realtime.fastContext,
+            args,
+            logger: log,
+          });
+          if (fastContext.handled) {
+            return fastContext.result;
+          }
+          const { provider: agentProvider, model } = resolveVoiceResponseModel({
+            voiceConfig: effectiveConfig,
+            agentRuntime,
+          });
+          const thinkLevel = agentRuntime.resolveThinkingDefault({
+            cfg,
+            provider: agentProvider,
+            model,
+          });
+          return await consultRealtimeVoiceAgent({
+            cfg,
+            agentRuntime,
+            logger: log,
+            agentId,
+            sessionKey,
+            messageProvider: "voice",
+            lane: "voice",
+            runIdPrefix: `voice-realtime-consult:${callId}`,
+            args,
+            transcript: mapVoiceCallConsultTranscript(call, handlerContext),
+            surface: "a live phone call",
+            userLabel: "Caller",
+            assistantLabel: "Agent",
+            questionSourceLabel: "caller",
+            provider: agentProvider,
+            model,
+            thinkLevel,
+            timeoutMs: effectiveConfig.responseTimeoutMs,
+            toolsAllow: resolveRealtimeVoiceAgentConsultToolsAllow(
+              effectiveConfig.realtime.toolPolicy,
+            ),
+            extraSystemPrompt: REALTIME_VOICE_CONSULT_SYSTEM_PROMPT,
+          });
+        },
+      );
+    }
+    webhookServer.setRealtimeHandler(realtimeHandler);
+  }
   const lifecycle = createRuntimeResourceLifecycle({ config, webhookServer });
 
   const localUrl = await webhookServer.start();
@@ -190,9 +423,7 @@ export async function createVoiceCallRuntime(params: {
         lifecycle.setTunnelResult(nextTunnelResult);
         publicUrl = nextTunnelResult?.publicUrl ?? null;
       } catch (err) {
-        log.error(
-          `[voice-call] Tunnel setup failed: ${err instanceof Error ? err.message : String(err)}`,
-        );
+        log.error(`[voice-call] Tunnel setup failed: ${formatErrorMessage(err)}`);
       }
     }
 
@@ -202,9 +433,23 @@ export async function createVoiceCallRuntime(params: {
 
     const webhookUrl = publicUrl ?? localUrl;
 
+    if (
+      providerRequiresPublicWebhook(provider.name) &&
+      isProviderUnreachableWebhookUrl(webhookUrl)
+    ) {
+      throw new Error(
+        `[voice-call] ${provider.name} requires a publicly reachable webhook URL. ` +
+          `Refusing to use local-only webhook ${webhookUrl}. ` +
+          "Set plugins.entries.voice-call.config.publicUrl or enable tunnel/tailscale exposure.",
+      );
+    }
+
     if (publicUrl && provider.name === "twilio") {
       (provider as TwilioProvider).setPublicUrl(publicUrl);
     }
+    if (publicUrl && realtimeProvider) {
+      webhookServer.getRealtimeHandler()?.setPublicUrl(publicUrl);
+    }
 
     if (provider.name === "twilio" && config.streaming?.enabled) {
       const twilioProvider = provider as TwilioProvider;
@@ -214,15 +459,12 @@ export async function createVoiceCallRuntime(params: {
             coreConfig,
             ttsOverride: config.tts,
             runtime: ttsRuntime,
+            logger: log,
           });
           twilioProvider.setTTSProvider(ttsProvider);
           log.info("[voice-call] Telephony TTS provider configured");
         } catch (err) {
-          log.warn(
-            `[voice-call] Failed to initialize telephony TTS: ${
-              err instanceof Error ? err.message : String(err)
-            }`,
-          );
+          log.warn(`[voice-call] Failed to initialize telephony TTS: ${formatErrorMessage(err)}`);
         }
       } else {
         log.warn("[voice-call] Telephony TTS unavailable; streaming TTS disabled");
@@ -235,13 +477,17 @@ export async function createVoiceCallRuntime(params: {
       }
     }
 
+    if (realtimeProvider) {
+      log.info(`[voice-call] Realtime voice provider: ${realtimeProvider.provider.id}`);
+    }
+
     await manager.initialize(provider, webhookUrl);
 
     const stop = async () => await lifecycle.stop();
 
     log.info("[voice-call] Runtime initialized");
     log.info(`[voice-call] Webhook URL: ${webhookUrl}`);
-    if (publicUrl) {
+    if (publicUrl && publicUrl !== webhookUrl) {
       log.info(`[voice-call] Public URL: ${publicUrl}`);
     }
 

package/src/telephony-audio.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vitest";
+import { convertPcmToMulaw8k, resamplePcmTo8k } from "./telephony-audio.js";
+
+function makeSinePcm(
+  sampleRate: number,
+  frequencyHz: number,
+  durationSeconds: number,
+  amplitude = 12_000,
+): Buffer {
+  const samples = Math.floor(sampleRate * durationSeconds);
+  const output = Buffer.alloc(samples * 2);
+  for (let i = 0; i < samples; i++) {
+    const value = Math.round(Math.sin((2 * Math.PI * frequencyHz * i) / sampleRate) * amplitude);
+    output.writeInt16LE(value, i * 2);
+  }
+  return output;
+}
+
+function rmsPcm(buffer: Buffer): number {
+  const samples = Math.floor(buffer.length / 2);
+  if (samples === 0) {
+    return 0;
+  }
+  let sum = 0;
+  for (let i = 0; i < samples; i++) {
+    const sample = buffer.readInt16LE(i * 2);
+    sum += sample * sample;
+  }
+  return Math.sqrt(sum / samples);
+}
+
+describe("telephony-audio resamplePcmTo8k", () => {
+  it("returns identical buffer for 8k input", () => {
+    const pcm8k = makeSinePcm(8_000, 1_000, 0.2);
+    const resampled = resamplePcmTo8k(pcm8k, 8_000);
+    expect(resampled).toBe(pcm8k);
+  });
+
+  it("preserves low-frequency speech-band energy when downsampling", () => {
+    const input = makeSinePcm(48_000, 1_000, 0.6);
+    const output = resamplePcmTo8k(input, 48_000);
+    expect(output.length).toBe(9_600);
+    expect(rmsPcm(output)).toBeGreaterThan(7_500);
+  });
+
+  it("attenuates out-of-band high frequencies before 8k telephony conversion", () => {
+    const lowTone = resamplePcmTo8k(makeSinePcm(48_000, 1_000, 0.6), 48_000);
+    const highTone = resamplePcmTo8k(makeSinePcm(48_000, 6_000, 0.6), 48_000);
+    const ratio = rmsPcm(highTone) / rmsPcm(lowTone);
+    expect(ratio).toBeLessThan(0.1);
+  });
+});
+
+describe("telephony-audio convertPcmToMulaw8k", () => {
+  it("converts to 8k mu-law frame length", () => {
+    const input = makeSinePcm(24_000, 1_000, 0.5);
+    const mulaw = convertPcmToMulaw8k(input, 24_000);
+    // 0.5s @ 8kHz => 4000 8-bit samples
+    expect(mulaw.length).toBe(4_000);
+  });
+});

package/src/telephony-audio.ts

@@ -1,60 +1,4 @@
-const TELEPHONY_SAMPLE_RATE = 8000;
-
-function clamp16(value: number): number {
-  return Math.max(-32768, Math.min(32767, value));
-}
-
-/**
- * Resample 16-bit PCM (little-endian mono) to 8kHz using linear interpolation.
- */
-export function resamplePcmTo8k(input: Buffer, inputSampleRate: number): Buffer {
-  if (inputSampleRate === TELEPHONY_SAMPLE_RATE) {
-    return input;
-  }
-  const inputSamples = Math.floor(input.length / 2);
-  if (inputSamples === 0) {
-    return Buffer.alloc(0);
-  }
-
-  const ratio = inputSampleRate / TELEPHONY_SAMPLE_RATE;
-  const outputSamples = Math.floor(inputSamples / ratio);
-  const output = Buffer.alloc(outputSamples * 2);
-
-  for (let i = 0; i < outputSamples; i++) {
-    const srcPos = i * ratio;
-    const srcIndex = Math.floor(srcPos);
-    const frac = srcPos - srcIndex;
-
-    const s0 = input.readInt16LE(srcIndex * 2);
-    const s1Index = Math.min(srcIndex + 1, inputSamples - 1);
-    const s1 = input.readInt16LE(s1Index * 2);
-
-    const sample = Math.round(s0 + frac * (s1 - s0));
-    output.writeInt16LE(clamp16(sample), i * 2);
-  }
-
-  return output;
-}
-
-/**
- * Convert 16-bit PCM to 8-bit mu-law (G.711).
- */
-export function pcmToMulaw(pcm: Buffer): Buffer {
-  const samples = Math.floor(pcm.length / 2);
-  const mulaw = Buffer.alloc(samples);
-
-  for (let i = 0; i < samples; i++) {
-    const sample = pcm.readInt16LE(i * 2);
-    mulaw[i] = linearToMulaw(sample);
-  }
-
-  return mulaw;
-}
-
-export function convertPcmToMulaw8k(pcm: Buffer, inputSampleRate: number): Buffer {
-  const pcm8k = resamplePcmTo8k(pcm, inputSampleRate);
-  return pcmToMulaw(pcm8k);
-}
+export { convertPcmToMulaw8k, resamplePcmTo8k } from "openclaw/plugin-sdk/realtime-voice";
 
 /**
  * Chunk audio buffer into 20ms frames for streaming (8kHz mono mu-law).
@@ -66,25 +10,3 @@ export function chunkAudio(audio: Buffer, chunkSize = 160): Generator<Buffer, vo
     }
   })();
 }
-
-function linearToMulaw(sample: number): number {
-  const BIAS = 132;
-  const CLIP = 32635;
-
-  const sign = sample < 0 ? 0x80 : 0;
-  if (sample < 0) {
-    sample = -sample;
-  }
-  if (sample > CLIP) {
-    sample = CLIP;
-  }
-
-  sample += BIAS;
-  let exponent = 7;
-  for (let expMask = 0x4000; (sample & expMask) === 0 && exponent > 0; exponent--) {
-    expMask >>= 1;
-  }
-
-  const mantissa = (sample >> (exponent + 3)) & 0x0f;
-  return ~(sign | (exponent << 4) | mantissa) & 0xff;
-}