@pimote/pimote 0.1.1 → 0.3.0

package/server/dist/voice/walk-back.js

@@ -0,0 +1,198 @@
+// Walkback rewrite: surgical truncation of conversation history when
+// speechmux reports the user heard only a prefix of an assistant
+// utterance.
+//
+// **Identity-based design.** Walkback targets a specific `speak()` tool
+// call by its `toolCallId`. That id is round-tripped through speechmux
+// (every outgoing `token`/`end` frame carries it; speechmux echoes it
+// back on `rollback`/`abort`) so we know exactly which utterance the
+// `heardText` belongs to. The previous design used a captured snapshot
+// of the in-flight assistant message and a string-prefix-matching
+// algorithm — both of which broke whenever a turn contained more than
+// one speak() or whenever the snapshot drifted out of sync with the
+// real conversation.
+//
+// **Contract:** see `docs/plans/voice-mode.md` for the high-level
+// behavioural spec. Briefly:
+//
+//   1. The trailing pi-synthetic empty-text aborted assistant (if any)
+//      is always stripped, even when no rollback is pending. This is
+//      pi's marker for "agent run was aborted"; we don't want it in
+//      the LLM context.
+//
+//   2. With a rollback pending, locate the speak block by
+//      `targetSpeakToolCallId`. If found:
+//        - If `heardText` is empty: drop the speak block entirely (and
+//          its paired tool_result if present).
+//        - If `heardText.length >= block.text.length`: keep block as-is
+//          (whole utterance was heard).
+//        - Otherwise: replace the block's text with `heardText` and
+//          drop the paired tool_result.
+//      Then drop blocks AFTER the target in the same message, and drop
+//      any subsequent assistant/tool_result messages — none of those
+//      could have been heard if the user interrupted at the target.
+//
+//   3. If the target is not found in messages (e.g. compacted away),
+//      walkback is a no-op beyond step 1.
+//
+// **Content-block shape compatibility.** The function handles both
+// pi-agent-core's internal AgentMessage shape (`type:'toolCall'` +
+// `arguments`) and the Anthropic API shape (`type:'tool_use'` +
+// `input`). Earlier versions only matched the latter, which silently
+// failed on every real captured message.
+/**
+ * Apply walkback against `messages`. Pure function.
+ *
+ * Returns a new array; never mutates the input.
+ */
+export function walkBack(input) {
+    const stripped = stripTrailingAbortedEmpty(input.messages);
+    if (input.rollback === null)
+        return stripped;
+    return rewriteByToolCallId(stripped, input.rollback.heardText, input.rollback.targetSpeakToolCallId);
+}
+// ---------------------------------------------------------------------------
+/** True for the synthetic assistant pi appends to state on abort. */
+export function isAbortedEmptyAssistant(msg) {
+    if (!isAssistantMessage(msg))
+        return false;
+    if (stopReason(msg) !== 'aborted')
+        return false;
+    return isEmptyText(contentOf(msg));
+}
+function stripTrailingAbortedEmpty(messages) {
+    let cut = messages.length;
+    while (cut > 0 && isAbortedEmptyAssistant(messages[cut - 1]))
+        cut -= 1;
+    return cut === messages.length ? messages.slice() : messages.slice(0, cut);
+}
+function rewriteByToolCallId(messages, heardText, targetId) {
+    // Search from the back — toolCallIds are unique per session, so the
+    // first match is the right one, but searching backward minimises work
+    // for the common case (target is in the recent tail).
+    let targetMsgIdx = -1;
+    let targetBlockIdx = -1;
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const msg = messages[i];
+        if (!isAssistantMessage(msg))
+            continue;
+        const content = contentOf(msg);
+        for (let j = 0; j < content.length; j++) {
+            if (isSpeakToolCall(content[j]) && getToolCallId(content[j]) === targetId) {
+                targetMsgIdx = i;
+                targetBlockIdx = j;
+                break;
+            }
+        }
+        if (targetMsgIdx !== -1)
+            break;
+    }
+    if (targetMsgIdx === -1) {
+        // Target gone (compacted, or never landed in messages). Best we can
+        // do is honour step 1 (already done).
+        return messages;
+    }
+    const targetMsg = messages[targetMsgIdx];
+    const targetContent = contentOf(targetMsg);
+    const targetBlock = targetContent[targetBlockIdx];
+    const originalText = getSpeakText(targetBlock);
+    const newBlocks = targetContent.slice(0, targetBlockIdx);
+    const droppedToolUseIds = new Set();
+    if (heardText.length === 0) {
+        // Nothing was heard of this speak. Drop the block and its paired
+        // tool_result (if any).
+        droppedToolUseIds.add(targetId);
+    }
+    else if (heardText.length >= originalText.length) {
+        // Entire utterance was heard. Keep block intact.
+        newBlocks.push(targetBlock);
+    }
+    else {
+        // Partial. Truncate text in-place and drop the paired tool_result
+        // (per the contract — a truncated speak's result is no longer
+        // grounded in what the user heard).
+        newBlocks.push(replaceSpeakText(targetBlock, heardText));
+        droppedToolUseIds.add(targetId);
+    }
+    // Anything in this message AFTER the target block was emitted after
+    // the heard prefix and so was not heard.
+    for (let j = targetBlockIdx + 1; j < targetContent.length; j++) {
+        const id = getToolCallId(targetContent[j]);
+        if (id)
+            droppedToolUseIds.add(id);
+    }
+    const rewrittenTarget = {
+        ...targetMsg,
+        content: newBlocks,
+        stopReason: 'aborted',
+    };
+    // Anything AFTER the target message in the array was emitted by the
+    // agent after the interrupted speak — drop it. This includes any
+    // tool_result messages whose paired speak we just truncated, plus
+    // any subsequent assistant messages.
+    return [...messages.slice(0, targetMsgIdx), rewrittenTarget];
+}
+// ---------------------------------------------------------------------------
+// Shape-tolerant accessors. pi-agent-core's runtime AgentMessage uses
+// `toolCall`/`arguments`; the Anthropic API shape uses `tool_use`/`input`.
+// Tests / tooling may pass either; we accept both.
+function isAssistantMessage(msg) {
+    return msg.role === 'assistant';
+}
+function stopReason(msg) {
+    return msg.stopReason;
+}
+function contentOf(msg) {
+    const c = msg.content;
+    return Array.isArray(c) ? c : [];
+}
+function isEmptyText(blocks) {
+    if (blocks.length === 0)
+        return true;
+    return blocks.every((b) => {
+        if (b.type !== 'text')
+            return false;
+        const t = b.text;
+        return typeof t === 'string' && t.trim() === '';
+    });
+}
+export function isSpeakToolCall(block) {
+    if (block.type !== 'toolCall' && block.type !== 'tool_use')
+        return false;
+    return block.name === 'speak';
+}
+function getToolCallId(block) {
+    const id = block.id;
+    return typeof id === 'string' ? id : undefined;
+}
+function getSpeakText(block) {
+    // Try both shapes; whichever holds a string wins.
+    const args = block.arguments;
+    if (args && typeof args === 'object') {
+        const t = args.text;
+        if (typeof t === 'string')
+            return t;
+    }
+    const input = block.input;
+    if (input && typeof input === 'object') {
+        const t = input.text;
+        if (typeof t === 'string')
+            return t;
+    }
+    return '';
+}
+function replaceSpeakText(block, text) {
+    // Preserve whichever args/input shape was present, replacing only the
+    // `text` field. We don't normalise to a single shape — that would
+    // diverge from whatever pi-agent-core/the provider expects.
+    const args = block.arguments;
+    const input = block.input;
+    if (args && typeof args === 'object') {
+        return { ...block, arguments: { ...args, text } };
+    }
+    if (input && typeof input === 'object') {
+        return { ...block, input: { ...input, text } };
+    }
+    // Neither shape present — set both defensively.
+    return { ...block, arguments: { text }, input: { text } };
+}

package/server/dist/voice-orchestrator-boot.js

@@ -0,0 +1,90 @@
+// Wire the VoiceOrchestrator together with its runtime dependencies at
+// server boot time. Kept separate from `index.ts` so the wiring is
+// testable (no network / child_process side-effects at import time) and
+// isolated from the plain HTTP/WS boot sequence.
+import { spawn } from 'node:child_process';
+import { VoiceOrchestrator } from './voice-orchestrator.js';
+/**
+ * Construct a VoiceOrchestrator backed by real seams:
+ * - speechmux sidecar via `child_process.spawn`
+ * - displacement = looks up current owner via clientRegistry and calls its
+ *   `sendDisplacedEvent(sessionId)`
+ *
+ * Auth on `/signal` is handled by Cloudflare Access at the edge, and
+ * per-session TURN credentials are minted by speechmux and returned to the
+ * PWA in its `/signal` `session` response. Pimote's orchestrator only
+ * hands out the signalling URL.
+ */
+export function buildVoiceOrchestrator(args) {
+    const { config, sessionManager, clientRegistry } = args;
+    let speechmuxProc = null;
+    const busResolver = {
+        getSlot: (sessionId) => sessionManager.getSlot(sessionId),
+        getEventBus: (sessionId) => sessionManager.getSlot(sessionId)?.eventBusRef.current ?? null,
+    };
+    const orchestrator = new VoiceOrchestrator({
+        config,
+        sessionManager,
+        busResolver,
+        startSpeechmux: async () => {
+            const bin = config.voice?.speechmuxBinary;
+            if (!bin) {
+                console.log('[voice] speechmuxBinary not configured; assuming speechmux is externally managed (systemd, container, remote host, etc.)');
+                return;
+            }
+            if (speechmuxProc)
+                return;
+            speechmuxProc = spawn(bin, [], { stdio: ['ignore', 'inherit', 'inherit'] });
+            speechmuxProc.on('exit', (code, signal) => {
+                console.warn(`[voice] speechmux exited (code=${code}, signal=${signal})`);
+                speechmuxProc = null;
+            });
+            // NB: we do not wait for a ready marker here — speechmux emits readiness
+            // to its own logs. Callers should ensure startup ordering or implement a
+            // readiness probe as part of the Step 14 smoke.
+        },
+        stopSpeechmux: async () => {
+            if (!speechmuxProc)
+                return;
+            const proc = speechmuxProc;
+            speechmuxProc = null;
+            await new Promise((resolve) => {
+                const timer = setTimeout(() => {
+                    try {
+                        proc.kill('SIGKILL');
+                    }
+                    catch {
+                        /* ignore */
+                    }
+                    resolve();
+                }, 2000);
+                proc.once('exit', () => {
+                    clearTimeout(timer);
+                    resolve();
+                });
+                try {
+                    proc.kill('SIGTERM');
+                }
+                catch {
+                    clearTimeout(timer);
+                    resolve();
+                }
+            });
+        },
+        displaceOwner: async (sessionId, _newOwner) => {
+            const slot = sessionManager.getSlot(sessionId);
+            const existingClientId = slot?.connection?.connectedClientId;
+            if (!existingClientId)
+                return;
+            const existing = clientRegistry.get(existingClientId);
+            existing?.sendDisplacedEvent(sessionId);
+        },
+        isOwnedByVoiceCall: (sessionId) => orchestrator.isCallActive(sessionId),
+    });
+    return {
+        orchestrator,
+        shutdown: async () => {
+            await orchestrator.stop();
+        },
+    };
+}

package/server/dist/voice-orchestrator.js

@@ -0,0 +1,91 @@
+// Voice orchestrator — owns the speechmux sidecar lifecycle and the per-call
+// bind dispatch. See docs/plans/voice-mode.md → "Voice orchestrator".
+//
+// This file defines the interface surface + a stub implementation. The impl
+// phase fills in start()/stop()/bindCall()/endCall() bodies.
+/** Typed error carrying the discriminable reason code used in PimoteResponse.error. */
+export class CallBindError extends Error {
+    code;
+    constructor(code, message) {
+        super(message ?? code);
+        this.code = code;
+        this.name = 'CallBindError';
+    }
+}
+export class VoiceOrchestrator {
+    opts;
+    started = false;
+    activeCalls = new Set();
+    constructor(opts) {
+        this.opts = opts;
+    }
+    /** Spawns speechmux sidecar. Throws if it fails to start. */
+    async start() {
+        if (this.started)
+            return;
+        await this.opts.startSpeechmux();
+        this.started = true;
+    }
+    /** Kills speechmux. Idempotent. */
+    async stop() {
+        if (!this.started)
+            return;
+        await this.opts.stopSpeechmux();
+        this.started = false;
+        this.activeCalls.clear();
+    }
+    /** Called by ws-handler for CallBindCommand. */
+    async bindCall(args) {
+        const slot = this.opts.busResolver.getSlot(args.sessionId);
+        if (!slot) {
+            throw new CallBindError('call_bind_failed_session_not_found', `No session ${args.sessionId}`);
+        }
+        const alreadyOwned = this.opts.isOwnedByVoiceCall(args.sessionId);
+        if (alreadyOwned && !args.force) {
+            throw new CallBindError('call_bind_failed_owned', 'Session already bound to a voice call');
+        }
+        if (alreadyOwned && args.force) {
+            await this.opts.displaceOwner(args.sessionId, args.clientConnection);
+        }
+        // Voice-disabled guard: if speechmux wiring isn't configured, fail the
+        // bind here rather than handing the client empty URLs. Speechmux is
+        // what mints the per-call TURN creds now (in the /signal `session`
+        // response) and what authenticates peers (via Cloudflare Access at the
+        // edge), so pimote no longer needs to mint anything.
+        const signalUrl = this.opts.config.voice?.speechmuxSignalUrl;
+        const llmWsUrl = this.opts.config.voice?.speechmuxLlmWsUrl;
+        if (!signalUrl || !llmWsUrl) {
+            throw new CallBindError('call_bind_failed_internal', 'voice_disabled: speechmux signal URL / llm WS URL not configured');
+        }
+        const bus = this.opts.busResolver.getEventBus(args.sessionId);
+        if (!bus) {
+            throw new CallBindError('call_bind_failed_internal', 'Session has no EventBus');
+        }
+        const activate = {
+            type: 'pimote:voice:activate',
+            sessionId: args.sessionId,
+            speechmuxWsUrl: llmWsUrl,
+        };
+        bus.emit(activate.type, activate);
+        this.activeCalls.add(args.sessionId);
+        return {
+            sessionId: args.sessionId,
+            webrtcSignalUrl: signalUrl,
+        };
+    }
+    /** Called by ws-handler for CallEndCommand, or internally on displacement/error. Idempotent. */
+    async endCall(args) {
+        if (!this.activeCalls.has(args.sessionId))
+            return;
+        this.activeCalls.delete(args.sessionId);
+        const bus = this.opts.busResolver.getEventBus(args.sessionId);
+        if (bus) {
+            const deactivate = { type: 'pimote:voice:deactivate', sessionId: args.sessionId };
+            bus.emit(deactivate.type, deactivate);
+        }
+    }
+    /** True if the given session currently has an active voice call bound. */
+    isCallActive(sessionId) {
+        return this.activeCalls.has(sessionId);
+    }
+}