@pouchy_ai/companion-sdk 0.10.0

package/dist/call.js

@@ -0,0 +1,478 @@
+// SDK-native voice call — opens a live WebRTC voice session directly to the
+// provider using the credentials from startCall(), with NO dependency on Pouchy's
+// first-party connection wrappers (which are coupled to app stores / genui). This
+// is what lets a pure third-party embed do voice.
+//
+// - OpenAI Realtime: fully self-contained WebRTC (no deps).
+// - ElevenLabs Convai: dynamically imports '@elevenlabs/client' (an OPTIONAL peer
+//   dependency) only when that provider is selected; if it's not installed the
+//   call rejects with a clear message.
+//
+// Browser-only (needs WebRTC + getUserMedia). Returns a handle to hang up and to
+// inject live context mid-call (the bridge the server uses for voiceRelevant).
+/** Universal "host control" verbs the companion can call on ANY surface (web /
+ *  game / app / hardware), regardless of provider. They're generic on purpose —
+ *  the SPECIFIC intent rides in the string params, which YOUR surface interprets
+ *  — so a single fixed schema set serves every embed AND can be pre-registered on
+ *  the (otherwise static) ElevenLabs Convai agent. `invoke_action` is the
+ *  catch-all trigger; the SDK unwraps it to the named action so it reaches your
+ *  onToolCall just like a declared tool. The others pass through by name. */
+export const HOST_CONTROL_TOOLS = [
+    {
+        name: 'invoke_action',
+        description: 'Trigger a named action the current surface supports — e.g. pause, resume, start, stop, restart, next, previous, submit, confirm, cancel, refresh. Put the action name in "action" and any extra args in "params".',
+        parameters: {
+            type: 'object',
+            properties: {
+                action: { type: 'string', description: 'the action, e.g. "pause" / "resume" / "next"' },
+                params: { type: 'object', description: 'optional extra arguments for the action' }
+            },
+            required: ['action']
+        }
+    },
+    {
+        name: 'set_feature',
+        description: 'Turn a named feature/capability on or off (e.g. hints, captions, dark mode, a light, notifications, a mode).',
+        parameters: {
+            type: 'object',
+            properties: { feature: { type: 'string' }, on: { type: 'boolean' } },
+            required: ['feature', 'on']
+        }
+    },
+    {
+        name: 'set_value',
+        description: 'Set a named control to a value — volume, brightness, difficulty, speed, zoom, temperature, etc.',
+        parameters: {
+            type: 'object',
+            properties: {
+                name: { type: 'string' },
+                value: { description: 'the value: a number, string, or boolean' }
+            },
+            required: ['name', 'value']
+        }
+    },
+    {
+        name: 'navigate',
+        description: 'Go to a named view / screen / page / section / mode / level in the surface.',
+        parameters: {
+            type: 'object',
+            properties: { to: { type: 'string' }, params: { type: 'object' } },
+            required: ['to']
+        }
+    },
+    {
+        name: 'highlight',
+        description: "Draw the user's attention to a named element (guidance / onboarding / a tutorial step).",
+        parameters: {
+            type: 'object',
+            properties: { target: { type: 'string' }, note: { type: 'string' } },
+            required: ['target']
+        }
+    }
+];
+/** Names of the host-control verbs (for routing). */
+export const HOST_CONTROL_TOOL_NAMES = HOST_CONTROL_TOOLS.map((t) => t.name);
+/** Avatar gesture/expression tools the embodied ("形象") companion calls to emote
+ *  — drive a VRM/3D body's gestures and facial expression. These are offered on
+ *  EVERY embed (like the host verbs) so the companion can express itself without
+ *  each third party re-declaring them, and — crucially — so the SDK can register
+ *  a no-op handler for them on the ElevenLabs Convai path, where the shared agent
+ *  is pre-configured to call play_gesture / play_expression regardless of what the
+ *  embed declares. An integrator that actually renders an avatar opts in by
+ *  declaring (or handling) the same tool name and the SDK routes the call to them;
+ *  otherwise it answers with a silent no-op success (see invokeVoiceTool / emit),
+ *  so a pure audio or 2D embed can ignore avatars entirely and the companion's
+ *  emote calls never error. Enum values mirror the canonical GESTURE_CATEGORIES /
+ *  EXPRESSION_INTENTS the avatar runtime understands (kept inline so the SDK stays
+ *  self-contained). */
+export const AVATAR_VISUAL_TOOLS = [
+    {
+        name: 'play_gesture',
+        description: 'Play a full-body avatar gesture to express yourself physically (greeting, agreement, thinking, celebration, dance, …). No effect on surfaces without an avatar.',
+        parameters: {
+            type: 'object',
+            properties: {
+                category: {
+                    type: 'string',
+                    description: 'the kind of gesture',
+                    enum: [
+                        'greeting',
+                        'agreement',
+                        'disagreement',
+                        'thinking',
+                        'celebration',
+                        'affection',
+                        'shy',
+                        'surprise',
+                        'dance',
+                        'point',
+                        'pose'
+                    ]
+                }
+            },
+            required: ['category']
+        }
+    },
+    {
+        name: 'play_expression',
+        description: "Set the avatar's facial expression to convey emotion (smile, laugh, wink, sad, surprised, …). No effect on surfaces without an avatar.",
+        parameters: {
+            type: 'object',
+            properties: {
+                intent: {
+                    type: 'string',
+                    description: 'the facial expression',
+                    enum: [
+                        'smile',
+                        'laugh',
+                        'wink',
+                        'shy',
+                        'pout',
+                        'sad',
+                        'angry',
+                        'surprised',
+                        'relaxed',
+                        'smug',
+                        'neutral'
+                    ]
+                }
+            },
+            required: ['intent']
+        }
+    }
+];
+/** Names of the avatar visual tools (for routing + the no-op fallback). */
+export const AVATAR_VISUAL_TOOL_NAMES = AVATAR_VISUAL_TOOLS.map((t) => t.name);
+const OPENAI_REALTIME_CALLS = 'https://api.openai.com/v1/realtime/calls';
+/** Open a live call from pre-minted credentials. */
+export async function openCompanionCall(creds, opts = {}, bridge) {
+    if (typeof navigator === 'undefined' || !navigator.mediaDevices) {
+        throw new Error('connectCall is browser-only (needs WebRTC + microphone access)');
+    }
+    return creds.provider === 'openai-realtime'
+        ? openOpenAICall(creds, opts, bridge)
+        : openConvaiCall(creds, opts, bridge);
+}
+// ── OpenAI Realtime — self-contained WebRTC ─────────────────────────────────
+async function openOpenAICall(creds, opts, bridge) {
+    const pc = new RTCPeerConnection();
+    const audioEl = opts.audioElement ?? new Audio();
+    audioEl.autoplay = true;
+    pc.ontrack = (e) => {
+        audioEl.srcObject = e.streams[0];
+        void audioEl.play().catch(() => { });
+    };
+    const mic = await navigator.mediaDevices.getUserMedia({ audio: true });
+    for (const track of mic.getAudioTracks())
+        pc.addTrack(track, mic);
+    // Half-duplex mic (mirrors the first-party realtime path, #1524): on a phone
+    // SPEAKER browser AEC doesn't fully cancel the companion's own voice, so it
+    // leaks into the mic, is transcribed as a USER turn, and barges in on itself —
+    // the avatar cuts off mid-sentence after one line (field report: SDK voice on
+    // agentpoker "说一句就断了"). Mute the owned mic track WHILE the model speaks
+    // (sends silence, keeps the track + transport alive), re-open after a short tail
+    // when the turn ends. Tradeoff: no voice barge-in mid-utterance — far better than
+    // the companion interrupting itself.
+    const MIC_UNMUTE_TAIL_MS = 350;
+    let micUnmuteTimer = null;
+    const setMicEnabled = (enabled) => {
+        try {
+            for (const tr of mic.getAudioTracks())
+                tr.enabled = enabled;
+        }
+        catch {
+            /* track gone — ignore */
+        }
+    };
+    const muteMicForSpeech = () => {
+        if (micUnmuteTimer) {
+            clearTimeout(micUnmuteTimer);
+            micUnmuteTimer = null;
+        }
+        setMicEnabled(false);
+    };
+    const scheduleMicUnmute = () => {
+        if (micUnmuteTimer)
+            clearTimeout(micUnmuteTimer);
+        micUnmuteTimer = setTimeout(() => {
+            micUnmuteTimer = null;
+            if (!closed)
+                setMicEnabled(true);
+        }, MIC_UNMUTE_TAIL_MS);
+    };
+    const dc = pc.createDataChannel('oai-events');
+    // `response.function_call_arguments.done` carries the call_id + args but often
+    // not the name — the name arrives earlier on `response.output_item.added`; track it.
+    const callNames = new Map();
+    if (bridge?.tools.length) {
+        dc.addEventListener('open', () => {
+            try {
+                dc.send(JSON.stringify({
+                    type: 'session.update',
+                    session: {
+                        type: 'realtime',
+                        tool_choice: 'auto',
+                        tools: bridge.tools.map((t) => ({
+                            type: 'function',
+                            name: t.name,
+                            description: t.description ?? '',
+                            parameters: t.parameters ?? { type: 'object', properties: {} }
+                        }))
+                    }
+                }));
+            }
+            catch {
+                /* a rejected tools update must not kill the call */
+            }
+        });
+    }
+    dc.onmessage = (ev) => {
+        let msg;
+        try {
+            msg = JSON.parse(ev.data);
+        }
+        catch {
+            return;
+        }
+        // App tool-calling over voice: register name at start, dispatch at done.
+        if (bridge &&
+            msg.type === 'response.output_item.added' &&
+            msg.item?.type === 'function_call' &&
+            msg.item.call_id &&
+            msg.item.name) {
+            callNames.set(msg.item.call_id, msg.item.name);
+        }
+        if (bridge && msg.type === 'response.function_call_arguments.done' && msg.call_id) {
+            const callId = msg.call_id;
+            const name = (msg.name && msg.name) || callNames.get(callId) || '';
+            callNames.delete(callId);
+            if (name && bridge.tools.some((t) => t.name === name)) {
+                const argsText = typeof msg.arguments === 'string' ? msg.arguments : '{}';
+                void bridge.invoke(name, argsText).then((output) => {
+                    try {
+                        dc.send(JSON.stringify({
+                            type: 'conversation.item.create',
+                            item: { type: 'function_call_output', call_id: callId, output }
+                        }));
+                        dc.send(JSON.stringify({ type: 'response.create' }));
+                    }
+                    catch {
+                        /* channel gone */
+                    }
+                });
+            }
+            return;
+        }
+        if (msg.type === 'response.audio_transcript.done' && msg.transcript) {
+            opts.onTranscript?.({ role: 'assistant', text: msg.transcript });
+        }
+        else if (msg.type === 'conversation.item.input_audio_transcription.completed' &&
+            msg.transcript) {
+            opts.onTranscript?.({ role: 'user', text: msg.transcript });
+        }
+        else if (msg.type === 'output_audio_buffer.started') {
+            opts.onSpeakingChange?.(true);
+            // Model started speaking → mute the mic so its own audio can't
+            // echo-transcribe into a self-barge-in.
+            muteMicForSpeech();
+        }
+        else if (msg.type === 'output_audio_buffer.stopped' ||
+            msg.type === 'output_audio_buffer.cleared' ||
+            msg.type === 'response.done' ||
+            msg.type === 'conversation.interrupted') {
+            // Turn ended (or was cancelled / interrupted) — clear the speaking state
+            // and re-open the mic after a short tail so the last bit of the model's
+            // audio doesn't get captured the instant we unmute. Several event names
+            // can end a turn; any of them re-arms the mic.
+            opts.onSpeakingChange?.(false);
+            scheduleMicUnmute();
+        }
+    };
+    const offer = await pc.createOffer();
+    await pc.setLocalDescription(offer);
+    const res = await fetch(OPENAI_REALTIME_CALLS, {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${creds.clientSecret}`, 'Content-Type': 'application/sdp' },
+        body: offer.sdp
+    });
+    if (!res.ok) {
+        pc.close();
+        mic.getTracks().forEach((t) => t.stop());
+        throw new Error(`OpenAI Realtime SDP exchange failed (${res.status})`);
+    }
+    await pc.setRemoteDescription({ type: 'answer', sdp: await res.text() });
+    let closed = false;
+    const teardown = () => {
+        if (closed)
+            return;
+        closed = true;
+        if (micUnmuteTimer) {
+            clearTimeout(micUnmuteTimer);
+            micUnmuteTimer = null;
+        }
+        try {
+            dc.close();
+        }
+        catch {
+            /* ignore */
+        }
+        pc.close();
+        mic.getTracks().forEach((t) => t.stop());
+    };
+    pc.onconnectionstatechange = () => {
+        if (['failed', 'disconnected', 'closed'].includes(pc.connectionState)) {
+            // Tear down BEFORE surfacing the error. On a silent transport drop
+            // (Wi-Fi→cellular, ICE failure, server expiry) the embedder may never
+            // call close(), so without this the mic keeps recording indefinitely.
+            // Mirrors the first-party realtime/connection.ts fail() ordering.
+            teardown();
+            opts.onError?.(new Error(`call ${pc.connectionState}`));
+        }
+    };
+    return {
+        provider: 'openai-realtime',
+        close: teardown,
+        injectEvent: (text, speak = false) => {
+            if (closed || dc.readyState !== 'open' || !text.trim())
+                return;
+            dc.send(JSON.stringify({
+                type: 'conversation.item.create',
+                item: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] }
+            }));
+            if (speak)
+                dc.send(JSON.stringify({ type: 'response.create' }));
+        }
+    };
+}
+// ── ElevenLabs Convai — via the optional @elevenlabs/client peer dep ─────────
+async function openConvaiCall(creds, opts, bridge) {
+    let mod;
+    try {
+        mod = await import('@elevenlabs/client');
+    }
+    catch {
+        throw new Error("elevenlabs-convai call needs the optional peer dependency '@elevenlabs/client' — install it (npm i @elevenlabs/client)");
+    }
+    // App tools → EL client tools. NOTE: ElevenLabs only CALLS a client tool the
+    // agent already knows (declared on the agent config); a shared Convai agent
+    // can't take arbitrary per-app tools at runtime, so dynamic app tools are
+    // reliable on the OpenAI Realtime path. Registering the handlers is harmless
+    // and works for any tool the agent does know.
+    const clientTools = {};
+    if (bridge?.tools.length) {
+        for (const t of bridge.tools) {
+            clientTools[t.name] = (params) => bridge.invoke(t.name, JSON.stringify(params ?? {}));
+        }
+    }
+    let closed = false;
+    // Half-duplex mic (mirrors the first-party path, #1524): mute while the agent
+    // speaks so its own voice can't echo-transcribe into a self-barge-in (the SDK
+    // voice "说一句就断了" report), re-open after a short tail. Convai exposes
+    // setMicMuted; guard it (older client builds may lack it).
+    const MIC_UNMUTE_TAIL_MS = 350;
+    let micUnmuteTimer = null;
+    const setConvMicMuted = (muted) => {
+        try {
+            conv.setMicMuted?.(muted);
+        }
+        catch {
+            /* not supported on this client build — ignore */
+        }
+    };
+    const conv = await mod.Conversation.startSession({
+        conversationToken: creds.token,
+        connectionType: 'webrtc',
+        ...(Object.keys(clientTools).length ? { clientTools } : {}),
+        overrides: {
+            agent: {
+                // The server-assembled persona + memory snapshot drives the EL agent.
+                prompt: { prompt: creds.instructions },
+                ...(creds.language ? { language: creds.language } : {}),
+                ...(creds.firstMessage !== undefined ? { firstMessage: creds.firstMessage } : {})
+            },
+            ...(creds.voice ? { tts: { voiceId: creds.voice } } : {})
+        },
+        onModeChange: ({ mode }) => {
+            if (closed)
+                return;
+            const speaking = mode === 'speaking';
+            opts.onSpeakingChange?.(speaking);
+            if (speaking) {
+                if (micUnmuteTimer) {
+                    clearTimeout(micUnmuteTimer);
+                    micUnmuteTimer = null;
+                }
+                setConvMicMuted(true);
+            }
+            else {
+                if (micUnmuteTimer)
+                    clearTimeout(micUnmuteTimer);
+                micUnmuteTimer = setTimeout(() => {
+                    micUnmuteTimer = null;
+                    if (!closed)
+                        setConvMicMuted(false);
+                }, MIC_UNMUTE_TAIL_MS);
+            }
+        },
+        onMessage: ({ message, role }) => {
+            const text = (message ?? '').trim();
+            if (!closed && text) {
+                opts.onTranscript?.({ role: role === 'agent' ? 'assistant' : 'user', text });
+            }
+        },
+        onError: (message) => {
+            if (closed)
+                return;
+            // End the session (releases the EL SDK's mic + hidden <audio>) before
+            // surfacing the error — a silent drop otherwise leaves the mic live
+            // because close() depends on the embedder calling it. closed=true first
+            // prevents re-entry via the endSession→onDisconnect path.
+            closed = true;
+            try {
+                void conv.endSession().catch(() => { });
+            }
+            catch {
+                /* session not yet established */
+            }
+            opts.onError?.(new Error(message));
+        },
+        onDisconnect: () => {
+            if (closed)
+                return;
+            closed = true;
+            try {
+                void conv.endSession().catch(() => { });
+            }
+            catch {
+                /* session not yet established */
+            }
+            opts.onError?.(new Error('call disconnected'));
+        }
+    });
+    return {
+        provider: 'elevenlabs-convai',
+        close: () => {
+            if (closed)
+                return;
+            closed = true;
+            if (micUnmuteTimer) {
+                clearTimeout(micUnmuteTimer);
+                micUnmuteTimer = null;
+            }
+            void conv.endSession().catch(() => { });
+        },
+        injectEvent: (text, speak = false) => {
+            if (closed || !text.trim())
+                return;
+            try {
+                if (speak)
+                    conv.sendUserMessage(text);
+                else
+                    conv.sendContextualUpdate(text);
+            }
+            catch {
+                /* channel not ready */
+            }
+        }
+    };
+}