@pimote/pimote 0.2.0 → 0.3.0

package/server/dist/voice/walk-back.js

@@ -0,0 +1,198 @@
+// Walkback rewrite: surgical truncation of conversation history when
+// speechmux reports the user heard only a prefix of an assistant
+// utterance.
+//
+// **Identity-based design.** Walkback targets a specific `speak()` tool
+// call by its `toolCallId`. That id is round-tripped through speechmux
+// (every outgoing `token`/`end` frame carries it; speechmux echoes it
+// back on `rollback`/`abort`) so we know exactly which utterance the
+// `heardText` belongs to. The previous design used a captured snapshot
+// of the in-flight assistant message and a string-prefix-matching
+// algorithm — both of which broke whenever a turn contained more than
+// one speak() or whenever the snapshot drifted out of sync with the
+// real conversation.
+//
+// **Contract:** see `docs/plans/voice-mode.md` for the high-level
+// behavioural spec. Briefly:
+//
+//   1. The trailing pi-synthetic empty-text aborted assistant (if any)
+//      is always stripped, even when no rollback is pending. This is
+//      pi's marker for "agent run was aborted"; we don't want it in
+//      the LLM context.
+//
+//   2. With a rollback pending, locate the speak block by
+//      `targetSpeakToolCallId`. If found:
+//        - If `heardText` is empty: drop the speak block entirely (and
+//          its paired tool_result if present).
+//        - If `heardText.length >= block.text.length`: keep block as-is
+//          (whole utterance was heard).
+//        - Otherwise: replace the block's text with `heardText` and
+//          drop the paired tool_result.
+//      Then drop blocks AFTER the target in the same message, and drop
+//      any subsequent assistant/tool_result messages — none of those
+//      could have been heard if the user interrupted at the target.
+//
+//   3. If the target is not found in messages (e.g. compacted away),
+//      walkback is a no-op beyond step 1.
+//
+// **Content-block shape compatibility.** The function handles both
+// pi-agent-core's internal AgentMessage shape (`type:'toolCall'` +
+// `arguments`) and the Anthropic API shape (`type:'tool_use'` +
+// `input`). Earlier versions only matched the latter, which silently
+// failed on every real captured message.
+/**
+ * Apply walkback against `messages`. Pure function.
+ *
+ * Returns a new array; never mutates the input.
+ */
+export function walkBack(input) {
+    const stripped = stripTrailingAbortedEmpty(input.messages);
+    if (input.rollback === null)
+        return stripped;
+    return rewriteByToolCallId(stripped, input.rollback.heardText, input.rollback.targetSpeakToolCallId);
+}
+// ---------------------------------------------------------------------------
+/** True for the synthetic assistant pi appends to state on abort. */
+export function isAbortedEmptyAssistant(msg) {
+    if (!isAssistantMessage(msg))
+        return false;
+    if (stopReason(msg) !== 'aborted')
+        return false;
+    return isEmptyText(contentOf(msg));
+}
+function stripTrailingAbortedEmpty(messages) {
+    let cut = messages.length;
+    while (cut > 0 && isAbortedEmptyAssistant(messages[cut - 1]))
+        cut -= 1;
+    return cut === messages.length ? messages.slice() : messages.slice(0, cut);
+}
+function rewriteByToolCallId(messages, heardText, targetId) {
+    // Search from the back — toolCallIds are unique per session, so the
+    // first match is the right one, but searching backward minimises work
+    // for the common case (target is in the recent tail).
+    let targetMsgIdx = -1;
+    let targetBlockIdx = -1;
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const msg = messages[i];
+        if (!isAssistantMessage(msg))
+            continue;
+        const content = contentOf(msg);
+        for (let j = 0; j < content.length; j++) {
+            if (isSpeakToolCall(content[j]) && getToolCallId(content[j]) === targetId) {
+                targetMsgIdx = i;
+                targetBlockIdx = j;
+                break;
+            }
+        }
+        if (targetMsgIdx !== -1)
+            break;
+    }
+    if (targetMsgIdx === -1) {
+        // Target gone (compacted, or never landed in messages). Best we can
+        // do is honour step 1 (already done).
+        return messages;
+    }
+    const targetMsg = messages[targetMsgIdx];
+    const targetContent = contentOf(targetMsg);
+    const targetBlock = targetContent[targetBlockIdx];
+    const originalText = getSpeakText(targetBlock);
+    const newBlocks = targetContent.slice(0, targetBlockIdx);
+    const droppedToolUseIds = new Set();
+    if (heardText.length === 0) {
+        // Nothing was heard of this speak. Drop the block and its paired
+        // tool_result (if any).
+        droppedToolUseIds.add(targetId);
+    }
+    else if (heardText.length >= originalText.length) {
+        // Entire utterance was heard. Keep block intact.
+        newBlocks.push(targetBlock);
+    }
+    else {
+        // Partial. Truncate text in-place and drop the paired tool_result
+        // (per the contract — a truncated speak's result is no longer
+        // grounded in what the user heard).
+        newBlocks.push(replaceSpeakText(targetBlock, heardText));
+        droppedToolUseIds.add(targetId);
+    }
+    // Anything in this message AFTER the target block was emitted after
+    // the heard prefix and so was not heard.
+    for (let j = targetBlockIdx + 1; j < targetContent.length; j++) {
+        const id = getToolCallId(targetContent[j]);
+        if (id)
+            droppedToolUseIds.add(id);
+    }
+    const rewrittenTarget = {
+        ...targetMsg,
+        content: newBlocks,
+        stopReason: 'aborted',
+    };
+    // Anything AFTER the target message in the array was emitted by the
+    // agent after the interrupted speak — drop it. This includes any
+    // tool_result messages whose paired speak we just truncated, plus
+    // any subsequent assistant messages.
+    return [...messages.slice(0, targetMsgIdx), rewrittenTarget];
+}
+// ---------------------------------------------------------------------------
+// Shape-tolerant accessors. pi-agent-core's runtime AgentMessage uses
+// `toolCall`/`arguments`; the Anthropic API shape uses `tool_use`/`input`.
+// Tests / tooling may pass either; we accept both.
+function isAssistantMessage(msg) {
+    return msg.role === 'assistant';
+}
+function stopReason(msg) {
+    return msg.stopReason;
+}
+function contentOf(msg) {
+    const c = msg.content;
+    return Array.isArray(c) ? c : [];
+}
+function isEmptyText(blocks) {
+    if (blocks.length === 0)
+        return true;
+    return blocks.every((b) => {
+        if (b.type !== 'text')
+            return false;
+        const t = b.text;
+        return typeof t === 'string' && t.trim() === '';
+    });
+}
+export function isSpeakToolCall(block) {
+    if (block.type !== 'toolCall' && block.type !== 'tool_use')
+        return false;
+    return block.name === 'speak';
+}
+function getToolCallId(block) {
+    const id = block.id;
+    return typeof id === 'string' ? id : undefined;
+}
+function getSpeakText(block) {
+    // Try both shapes; whichever holds a string wins.
+    const args = block.arguments;
+    if (args && typeof args === 'object') {
+        const t = args.text;
+        if (typeof t === 'string')
+            return t;
+    }
+    const input = block.input;
+    if (input && typeof input === 'object') {
+        const t = input.text;
+        if (typeof t === 'string')
+            return t;
+    }
+    return '';
+}
+function replaceSpeakText(block, text) {
+    // Preserve whichever args/input shape was present, replacing only the
+    // `text` field. We don't normalise to a single shape — that would
+    // diverge from whatever pi-agent-core/the provider expects.
+    const args = block.arguments;
+    const input = block.input;
+    if (args && typeof args === 'object') {
+        return { ...block, arguments: { ...args, text } };
+    }
+    if (input && typeof input === 'object') {
+        return { ...block, input: { ...input, text } };
+    }
+    // Neither shape present — set both defensively.
+    return { ...block, arguments: { text }, input: { text } };
+}

package/server/dist/voice-orchestrator-boot.js

@@ -0,0 +1,90 @@
+// Wire the VoiceOrchestrator together with its runtime dependencies at
+// server boot time. Kept separate from `index.ts` so the wiring is
+// testable (no network / child_process side-effects at import time) and
+// isolated from the plain HTTP/WS boot sequence.
+import { spawn } from 'node:child_process';
+import { VoiceOrchestrator } from './voice-orchestrator.js';
+/**
+ * Construct a VoiceOrchestrator backed by real seams:
+ * - speechmux sidecar via `child_process.spawn`
+ * - displacement = looks up current owner via clientRegistry and calls its
+ *   `sendDisplacedEvent(sessionId)`
+ *
+ * Auth on `/signal` is handled by Cloudflare Access at the edge, and
+ * per-session TURN credentials are minted by speechmux and returned to the
+ * PWA in its `/signal` `session` response. Pimote's orchestrator only
+ * hands out the signalling URL.
+ */
+export function buildVoiceOrchestrator(args) {
+    const { config, sessionManager, clientRegistry } = args;
+    let speechmuxProc = null;
+    const busResolver = {
+        getSlot: (sessionId) => sessionManager.getSlot(sessionId),
+        getEventBus: (sessionId) => sessionManager.getSlot(sessionId)?.eventBusRef.current ?? null,
+    };
+    const orchestrator = new VoiceOrchestrator({
+        config,
+        sessionManager,
+        busResolver,
+        startSpeechmux: async () => {
+            const bin = config.voice?.speechmuxBinary;
+            if (!bin) {
+                console.log('[voice] speechmuxBinary not configured; assuming speechmux is externally managed (systemd, container, remote host, etc.)');
+                return;
+            }
+            if (speechmuxProc)
+                return;
+            speechmuxProc = spawn(bin, [], { stdio: ['ignore', 'inherit', 'inherit'] });
+            speechmuxProc.on('exit', (code, signal) => {
+                console.warn(`[voice] speechmux exited (code=${code}, signal=${signal})`);
+                speechmuxProc = null;
+            });
+            // NB: we do not wait for a ready marker here — speechmux emits readiness
+            // to its own logs. Callers should ensure startup ordering or implement a
+            // readiness probe as part of the Step 14 smoke.
+        },
+        stopSpeechmux: async () => {
+            if (!speechmuxProc)
+                return;
+            const proc = speechmuxProc;
+            speechmuxProc = null;
+            await new Promise((resolve) => {
+                const timer = setTimeout(() => {
+                    try {
+                        proc.kill('SIGKILL');
+                    }
+                    catch {
+                        /* ignore */
+                    }
+                    resolve();
+                }, 2000);
+                proc.once('exit', () => {
+                    clearTimeout(timer);
+                    resolve();
+                });
+                try {
+                    proc.kill('SIGTERM');
+                }
+                catch {
+                    clearTimeout(timer);
+                    resolve();
+                }
+            });
+        },
+        displaceOwner: async (sessionId, _newOwner) => {
+            const slot = sessionManager.getSlot(sessionId);
+            const existingClientId = slot?.connection?.connectedClientId;
+            if (!existingClientId)
+                return;
+            const existing = clientRegistry.get(existingClientId);
+            existing?.sendDisplacedEvent(sessionId);
+        },
+        isOwnedByVoiceCall: (sessionId) => orchestrator.isCallActive(sessionId),
+    });
+    return {
+        orchestrator,
+        shutdown: async () => {
+            await orchestrator.stop();
+        },
+    };
+}

package/server/dist/voice-orchestrator.js

@@ -0,0 +1,91 @@
+// Voice orchestrator — owns the speechmux sidecar lifecycle and the per-call
+// bind dispatch. See docs/plans/voice-mode.md → "Voice orchestrator".
+//
+// This file defines the interface surface + a stub implementation. The impl
+// phase fills in start()/stop()/bindCall()/endCall() bodies.
+/** Typed error carrying the discriminable reason code used in PimoteResponse.error. */
+export class CallBindError extends Error {
+    code;
+    constructor(code, message) {
+        super(message ?? code);
+        this.code = code;
+        this.name = 'CallBindError';
+    }
+}
+export class VoiceOrchestrator {
+    opts;
+    started = false;
+    activeCalls = new Set();
+    constructor(opts) {
+        this.opts = opts;
+    }
+    /** Spawns speechmux sidecar. Throws if it fails to start. */
+    async start() {
+        if (this.started)
+            return;
+        await this.opts.startSpeechmux();
+        this.started = true;
+    }
+    /** Kills speechmux. Idempotent. */
+    async stop() {
+        if (!this.started)
+            return;
+        await this.opts.stopSpeechmux();
+        this.started = false;
+        this.activeCalls.clear();
+    }
+    /** Called by ws-handler for CallBindCommand. */
+    async bindCall(args) {
+        const slot = this.opts.busResolver.getSlot(args.sessionId);
+        if (!slot) {
+            throw new CallBindError('call_bind_failed_session_not_found', `No session ${args.sessionId}`);
+        }
+        const alreadyOwned = this.opts.isOwnedByVoiceCall(args.sessionId);
+        if (alreadyOwned && !args.force) {
+            throw new CallBindError('call_bind_failed_owned', 'Session already bound to a voice call');
+        }
+        if (alreadyOwned && args.force) {
+            await this.opts.displaceOwner(args.sessionId, args.clientConnection);
+        }
+        // Voice-disabled guard: if speechmux wiring isn't configured, fail the
+        // bind here rather than handing the client empty URLs. Speechmux is
+        // what mints the per-call TURN creds now (in the /signal `session`
+        // response) and what authenticates peers (via Cloudflare Access at the
+        // edge), so pimote no longer needs to mint anything.
+        const signalUrl = this.opts.config.voice?.speechmuxSignalUrl;
+        const llmWsUrl = this.opts.config.voice?.speechmuxLlmWsUrl;
+        if (!signalUrl || !llmWsUrl) {
+            throw new CallBindError('call_bind_failed_internal', 'voice_disabled: speechmux signal URL / llm WS URL not configured');
+        }
+        const bus = this.opts.busResolver.getEventBus(args.sessionId);
+        if (!bus) {
+            throw new CallBindError('call_bind_failed_internal', 'Session has no EventBus');
+        }
+        const activate = {
+            type: 'pimote:voice:activate',
+            sessionId: args.sessionId,
+            speechmuxWsUrl: llmWsUrl,
+        };
+        bus.emit(activate.type, activate);
+        this.activeCalls.add(args.sessionId);
+        return {
+            sessionId: args.sessionId,
+            webrtcSignalUrl: signalUrl,
+        };
+    }
+    /** Called by ws-handler for CallEndCommand, or internally on displacement/error. Idempotent. */
+    async endCall(args) {
+        if (!this.activeCalls.has(args.sessionId))
+            return;
+        this.activeCalls.delete(args.sessionId);
+        const bus = this.opts.busResolver.getEventBus(args.sessionId);
+        if (bus) {
+            const deactivate = { type: 'pimote:voice:deactivate', sessionId: args.sessionId };
+            bus.emit(deactivate.type, deactivate);
+        }
+    }
+    /** True if the given session currently has an active voice call bound. */
+    isCallActive(sessionId) {
+        return this.activeCalls.has(sessionId);
+    }
+}

package/server/dist/ws-handler.js

@@ -8,6 +8,7 @@ import { createExtensionUIBridge } from './extension-ui-bridge.js';
 import { findExternalPiProcesses, killExternalPiProcesses } from './takeover.js';
 import { mapAgentMessages, extractMessageEntryIds, applyEntryIds } from './message-mapper.js';
 import { getGitBranch } from './git-branch.js';
+import { CallBindError } from './voice-orchestrator.js';
 /** Parse data-URL encoded images into the shape the pi SDK expects. */
 function parseDataUrlImages(images) {
     if (!images || images.length === 0)
@@ -133,16 +134,18 @@ export class WsHandler {
     pushNotificationService;
     sessionMetadataStore;
     clientRegistry;
+    voiceOrchestrator;
     subscribedSessions = new Set();
     viewedSessionId = null;
     clientId;
-    constructor(sessionManager, folderIndex, ws, pushNotificationService, sessionMetadataStore, clientId, clientRegistry) {
+    constructor(sessionManager, folderIndex, ws, pushNotificationService, sessionMetadataStore, clientId, clientRegistry, voiceOrchestrator) {
         this.sessionManager = sessionManager;
         this.folderIndex = folderIndex;
         this.ws = ws;
         this.pushNotificationService = pushNotificationService;
         this.sessionMetadataStore = sessionMetadataStore;
         this.clientRegistry = clientRegistry;
+        this.voiceOrchestrator = voiceOrchestrator;
         this.clientId = clientId;
     }
     getViewedSessionId() {
@@ -480,6 +483,65 @@ export class WsHandler {
                     this.sendResponse(id, true, { sessionId: takeoverSessionId, killedProcesses: killedCount });
                     break;
                 }
+                // ---- Voice call control ----
+                case 'call_bind': {
+                    if (!this.voiceOrchestrator) {
+                        this.sendResponse(id, false, undefined, 'call_bind_failed_internal');
+                        break;
+                    }
+                    const slot = this.sessionManager.getSlot(command.sessionId);
+                    if (!slot) {
+                        this.sendResponse(id, false, undefined, 'call_bind_failed_session_not_found');
+                        break;
+                    }
+                    const connection = {
+                        ws: this.ws,
+                        connectedClientId: this.clientId,
+                        onSessionReset: (s) => this.handleSessionReset(s),
+                    };
+                    try {
+                        const data = await this.voiceOrchestrator.bindCall({
+                            sessionId: command.sessionId,
+                            clientConnection: connection,
+                            force: command.force ?? false,
+                        });
+                        this.sendResponse(id, true, data);
+                        this.sendEvent({ type: 'call_status', sessionId: command.sessionId, status: 'binding' });
+                    }
+                    catch (err) {
+                        if (err instanceof CallBindError) {
+                            this.sendResponse(id, false, undefined, err.code);
+                        }
+                        else {
+                            console.warn('[voice] call_bind failed', err);
+                            this.sendResponse(id, false, undefined, 'call_bind_failed_internal');
+                        }
+                    }
+                    break;
+                }
+                case 'call_end': {
+                    await this.voiceOrchestrator?.endCall({ sessionId: command.sessionId, reason: 'user_hangup' });
+                    this.sendResponse(id, true);
+                    this.sendEvent({ type: 'call_ended', sessionId: command.sessionId, reason: 'user_hangup' });
+                    break;
+                }
+                // ---- Client diagnostic logs (voice/call tracing) ----
+                case 'client_log': {
+                    // Forward to the server's logger so client-side traces interleave
+                    // with the server-side voice extension logs in the same journal.
+                    const clientWall = new Date(command.clientTimestampMs).toISOString();
+                    const serverWall = new Date().toISOString();
+                    const driftMs = Date.now() - command.clientTimestampMs;
+                    const line = `[voice_trace][client/${command.tag}] ${command.message} ${JSON.stringify({ clientWall, serverWall, driftMs, ...(command.data ?? {}) })}`;
+                    if (command.level === 'error')
+                        console.error(line);
+                    else if (command.level === 'warn')
+                        console.warn(line);
+                    else
+                        console.log(line);
+                    this.sendResponse(id, true);
+                    break;
+                }
                 // ---- Extension UI ----
                 case 'extension_ui_response': {
                     const uiSlot = command.sessionId ? this.sessionManager.getSession(command.sessionId) : undefined;
@@ -896,13 +958,26 @@ export class WsHandler {
         }
     }
     /** Notify the old owner that they've been displaced from a session.
-     *  No-op if the session is unowned or owned by this client. */
+     *  No-op if the session is unowned or owned by this client.
+     *
+     *  Voice-call tear-down on displacement lives in `sendDisplacedEvent` (the
+     *  old-owner-side site that also emits `call_ended { displaced }`), so this
+     *  method does not call `voiceOrchestrator.endCall` itself — see review
+     *  finding 4.
+     */
     displaceOwner(sessionId, slot) {
         if (slot.connection?.connectedClientId && slot.connection.connectedClientId !== this.clientId) {
             const oldHandler = this.clientRegistry.get(slot.connection.connectedClientId);
             if (oldHandler) {
                 oldHandler.sendDisplacedEvent(sessionId);
             }
+            else if (this.voiceOrchestrator?.isCallActive(sessionId)) {
+                // Stale owner id with no live handler — clean up orchestrator state
+                // so the new owner doesn't inherit a phantom active call.
+                this.voiceOrchestrator.endCall({ sessionId, reason: 'displaced' }).catch((err) => {
+                    console.warn('[voice] endCall on displace (stale handler) failed', err);
+                });
+            }
         }
     }
     /** Bind a slot to this client — sets ownership, WebSocket routing,
@@ -920,7 +995,9 @@ export class WsHandler {
         // ManagedSlot — on reconnect we skip rebinding, but on session reset
         // we must rebind so the bridge points at the new session state.
         if (!slot.sessionState.extensionsBound) {
-            const uiContext = createExtensionUIBridge(slot, this.pushNotificationService);
+            const uiContext = createExtensionUIBridge(slot, this.pushNotificationService, {
+                isVoiceModeActive: () => this.voiceOrchestrator?.isCallActive(sessionId) ?? false,
+            });
             const commandContextActions = createCommandContextActions(slot);
             await slot.session.bindExtensions({ uiContext, commandContextActions });
             slot.sessionState.extensionsBound = true;
@@ -939,9 +1016,12 @@ export class WsHandler {
             return;
         }
         // Session ID changed — rebuild session state in-place on the same slot.
-        const folderPath = slot.folderPath;
+        // rebuildSessionState refreshes slot.folderPath from the new session's header cwd,
+        // so capture folderPath AFTER the rebuild to pick up the new value (fork-from can
+        // change cwd, e.g. the worktree extension).
         // Rebuild session state (tears down old, creates new from runtime.session)
         this.sessionManager.rebuildSessionState(slot);
+        const folderPath = slot.folderPath;
         // Re-key the session map
         this.sessionManager.reKeySession(slot, oldSessionId, newSessionId);
         // Update handler bookkeeping
@@ -951,7 +1031,9 @@ export class WsHandler {
             this.viewedSessionId = newSessionId;
         }
         // Rebind extension UI bridge (new session state for dialog routing)
-        const uiContext = createExtensionUIBridge(slot, this.pushNotificationService);
+        const uiContext = createExtensionUIBridge(slot, this.pushNotificationService, {
+            isVoiceModeActive: () => this.voiceOrchestrator?.isCallActive(newSessionId) ?? false,
+        });
         const commandContextActions = createCommandContextActions(slot);
         await slot.session.bindExtensions({ uiContext, commandContextActions });
         slot.sessionState.extensionsBound = true;
@@ -1064,6 +1146,27 @@ export class WsHandler {
             sessionId,
             reason: 'displaced',
         });
+        // If the old owner had an active voice call on this session, tear down
+        // orchestrator bookkeeping and surface `call_ended { reason: 'displaced' }`
+        // so their VoiceCallStore tears down alongside the session_closed.
+        if (this.voiceOrchestrator?.isCallActive(sessionId)) {
+            void this.voiceOrchestrator.endCall({ sessionId, reason: 'displaced' });
+            this.sendEvent({
+                type: 'call_ended',
+                sessionId,
+                reason: 'displaced',
+            });
+        }
+    }
+    /** Broadcast a `call_ended` to this client (used by the session manager's
+     *  before-close hook so the orchestrator bookkeeping owner learns that a
+     *  server-initiated teardown happened). */
+    sendCallEndedEvent(sessionId, reason) {
+        this.sendEvent({
+            type: 'call_ended',
+            sessionId,
+            reason,
+        });
     }
     /** Send a session_closed event with reason 'killed' to this client's WebSocket.
      *  Also removes the session from this handler's subscribedSessions so that

package/shared/dist/index.d.ts

@@ -0,0 +1 @@
+export * from './protocol.js';

package/shared/dist/index.js

@@ -0,0 +1,2 @@
+// @pimote/shared barrel export
+export * from './protocol.js';