@pimote/pimote 0.6.0 → 0.8.0

package/server/dist/voice/index.js

@@ -83,10 +83,20 @@ export function createVoiceExtension(opts) {
         let state = initialState();
         let lastCtx = null;
         let speechmuxClient = null;
-        /** Slot read by the `context` hook to return rewritten messages. */
-        let pendingContextRewrite = null;
+        // Monotonic generation tag for the speechmux client. Bumped on every
+        // open_ws and close_ws so a discarded client's late callbacks (frame /
+        // disconnect) and an in-flight connect that the call already abandoned
+        // can detect they are stale and no-op. See voice lifecycle review (H1/H2).
+        let clientGeneration = 0;
+        // Detaches the current client's onFrame/onDisconnect listeners. Captured
+        // on open so we can unsubscribe when discarding the client.
+        let detachClientListeners = null;
         // ---- Reducer driver --------------------------------------------------
-        const dispatch = async (event) => {
+        // Synchronous core: reduce + write back state + trace. Returns the actions
+        // to execute. Split out from `dispatch` so the `context` hook can run a
+        // reduction and read the resulting rewrite synchronously — pi's context
+        // hook must return the rewritten messages inline, and `dispatch` is async. (M5)
+        const reduceAndApply = (event) => {
             const evtTrace = traceEvent(event);
             const lifecycleBefore = state.lifecycle.kind;
             const { next, actions } = reduce(state, event, {
@@ -101,6 +111,10 @@ export function createVoiceExtension(opts) {
                     blocks: blockSummary(state.message.blocks),
                 }));
             }
+            return actions;
+        };
+        const dispatch = async (event) => {
+            const actions = reduceAndApply(event);
             for (const action of actions) {
                 try {
                     await execute(action);
@@ -126,6 +140,19 @@ export function createVoiceExtension(opts) {
                     return;
                 }
                 case 'send_user_message': {
+                    // Steer-on-activate (deliverAs:'steer'): preserve in-flight work
+                    // rather than aborting it. When the agent is busy, inject the
+                    // message into the running turn (no abort); when idle there's no
+                    // turn to steer into, so send normally to trigger the turn. (M7)
+                    if (action.deliverAs === 'steer') {
+                        if (lastCtx && !lastCtx.isIdle()) {
+                            pi.sendUserMessage(action.text, { deliverAs: 'steer' });
+                        }
+                        else {
+                            pi.sendUserMessage(action.text);
+                        }
+                        return;
+                    }
                     // Ensure the agent is idle before sending. If it isn't, fire
                     // a synthesized barge-in (ctx.abort()) and wait for teardown
                     // — covers the case where the user spoke while the worker
@@ -154,7 +181,11 @@ export function createVoiceExtension(opts) {
                     return;
                 }
                 case 'open_ws': {
-                    // Reentrancy guard: close any prior client first.
+                    // Invalidate any prior client: bump the generation so its in-flight
+                    // callbacks no-op, detach its listeners, and close it.
+                    clientGeneration++;
+                    detachClientListeners?.();
+                    detachClientListeners = null;
                     try {
                         speechmuxClient?.close();
                     }
@@ -162,24 +193,53 @@ export function createVoiceExtension(opts) {
                         /* ignore */
                     }
                     speechmuxClient = null;
+                    const gen = clientGeneration;
                     try {
                         const client = await clientFactory({ wsUrl: action.url });
+                        // If the call ended (or another open started) while we were
+                        // connecting, this client is stale — close it and bail. (H2)
+                        if (gen !== clientGeneration) {
+                            try {
+                                client.close();
+                            }
+                            catch {
+                                /* ignore */
+                            }
+                            return;
+                        }
                         speechmuxClient = client;
-                        client.onFrame((frame) => {
+                        const offFrame = client.onFrame((frame) => {
+                            if (gen !== clientGeneration)
+                                return;
                             void dispatch({ type: 'ws:incoming', frame });
                         });
-                        client.onDisconnect(() => {
+                        const offDisconnect = client.onDisconnect(() => {
+                            if (gen !== clientGeneration)
+                                return;
                             void dispatch({ type: 'ws:disconnected' });
                         });
+                        detachClientListeners = () => {
+                            offFrame();
+                            offDisconnect();
+                        };
                         await dispatch({ type: 'ws:opened' });
                     }
                     catch (err) {
+                        // A stale open's failure is irrelevant — the FSM has moved on.
+                        if (gen !== clientGeneration)
+                            return;
                         console.warn('[voice] speechmux open failed', err);
                         await dispatch({ type: 'ws:open_failed', error: err });
                     }
                     return;
                 }
                 case 'close_ws': {
+                    // Bump the generation so a client still mid-connect (and any late
+                    // close/frame callbacks from the current one) is invalidated and
+                    // auto-cleaned. (H1/H2)
+                    clientGeneration++;
+                    detachClientListeners?.();
+                    detachClientListeners = null;
                     try {
                         speechmuxClient?.close();
                     }
@@ -213,19 +273,16 @@ export function createVoiceExtension(opts) {
                     return;
                 }
                 case 'emit_deactivate_request': {
-                    const sessionId = state.lifecycle.kind === 'active' || state.lifecycle.kind === 'activating' ? state.lifecycle.sessionId : '';
+                    // sessionId is carried on the action (populated by the reducer from
+                    // the pre-transition state) — reading it from `state` here would be
+                    // too late, since the lifecycle has already gone dormant. (M1)
                     const msg = {
                         type: 'pimote:voice:deactivate',
-                        sessionId,
+                        sessionId: action.sessionId,
                     };
                     pi.events.emit('pimote:voice:deactivate', msg);
                     return;
                 }
-                case 'rewrite_context': {
-                    // Stash; the `context` hook below reads this on its return.
-                    pendingContextRewrite = action.messages;
-                    return;
-                }
             }
         };
         // ---- EventBus listeners ---------------------------------------------
@@ -319,43 +376,42 @@ export function createVoiceExtension(opts) {
             }
         });
         pi.on('turn_end', (event) => {
-            if (state.lifecycle.kind !== 'active' || !speechmuxClient)
-                return;
+            // Route through the FSM so the floor_released frame is buffered when the
+            // greeting turn ends before the WS handshake completes, instead of being
+            // dropped. Only dispatch when the turn actually spoke. (M2)
             const lastSpeakResult = [...event.toolResults].reverse().find((result) => result.toolName === 'speak');
             if (!lastSpeakResult)
                 return;
-            try {
-                speechmuxClient.send(typeof lastSpeakResult.toolCallId === 'string' ? { type: 'floor_released', speak_id: lastSpeakResult.toolCallId } : { type: 'floor_released' });
-            }
-            catch (err) {
-                console.warn('[voice] speechmux send failed', 'floor_released', err);
-            }
+            void dispatch({
+                type: 'sdk:turn_end',
+                lastSpeakToolCallId: typeof lastSpeakResult.toolCallId === 'string' ? lastSpeakResult.toolCallId : null,
+            });
         });
         pi.on('agent_end', (event) => {
-            if (state.lifecycle.kind !== 'active' || !speechmuxClient)
-                return;
+            // Route through the FSM (same buffering rationale as turn_end). (M2)
             const error = event.error;
-            if (typeof error !== 'string' || error.length === 0)
-                return;
-            try {
-                speechmuxClient.send({ type: 'error', message: error });
-            }
-            catch (err) {
-                console.warn('[voice] speechmux send failed', 'error', err);
-            }
+            void dispatch({ type: 'sdk:agent_end', error: typeof error === 'string' && error.length > 0 ? error : null });
         });
         pi.on('context', (event, ctx) => {
             lastCtx = ctx;
-            // The walkback reducer always runs walkBack (even when no rewrite
-            // is pending — to strip aborted-empty-assistants). It writes the
-            // result into `pendingContextRewrite` via the `rewrite_context`
-            // action, which we read below.
-            void dispatch({ type: 'sdk:context', messages: event.messages });
-            const result = pendingContextRewrite;
-            pendingContextRewrite = null;
-            if (result)
-                return { messages: result };
-            return undefined;
+            // The walkback reducer always runs walkBack (even when no rewrite is
+            // pending — to strip aborted-empty assistants) and emits the result as a
+            // `rewrite_context` action. Drive the reduction synchronously and read
+            // the rewrite straight off the returned actions: no module-level slot,
+            // no dependence on action ordering or on `execute` staying side-effect
+            // free before the rewrite. (M5)
+            const actions = reduceAndApply({ type: 'sdk:context', messages: event.messages });
+            let rewritten;
+            for (const action of actions) {
+                if (action.kind === 'rewrite_context') {
+                    rewritten = action.messages;
+                }
+                else {
+                    // Any other side effects still run on the normal async path.
+                    void execute(action);
+                }
+            }
+            return rewritten ? { messages: rewritten } : undefined;
         });
     };
 }

package/server/dist/voice/walk-back.js

@@ -92,45 +92,97 @@ function rewriteByToolCallId(messages, heardText, targetId) {
         // do is honour step 1 (already done).
         return messages;
     }
+    // Walkback is about what the user *heard* (speech), not about undoing the
+    // agent's real work. So we surgically prune only speak() calls from the
+    // target onward — truncate/drop the target, drop later speaks — and keep
+    // every other tool call (and its result) and other content intact. Results
+    // of pruned speaks are dropped too (a tool_result with no tool_use is the
+    // riskier dangling direction). `stopReason` is preserved, not synthesised:
+    // the target is no longer necessarily the last message.
+    const droppedSpeakIds = new Set();
     const targetMsg = messages[targetMsgIdx];
     const targetContent = contentOf(targetMsg);
     const targetBlock = targetContent[targetBlockIdx];
     const originalText = getSpeakText(targetBlock);
+    // Blocks before the target were spoken/heard earlier in the turn — keep.
     const newBlocks = targetContent.slice(0, targetBlockIdx);
-    const droppedToolUseIds = new Set();
     if (heardText.length === 0) {
-        // Nothing was heard of this speak. Drop the block and its paired
-        // tool_result (if any).
-        droppedToolUseIds.add(targetId);
+        droppedSpeakIds.add(targetId); // nothing heard — drop the speak + its result
     }
     else if (heardText.length >= originalText.length) {
-        // Entire utterance was heard. Keep block intact.
-        newBlocks.push(targetBlock);
+        newBlocks.push(targetBlock); // whole utterance heard — keep intact (+ its result)
     }
     else {
-        // Partial. Truncate text in-place and drop the paired tool_result
-        // (per the contract — a truncated speak's result is no longer
-        // grounded in what the user heard).
-        newBlocks.push(replaceSpeakText(targetBlock, heardText));
-        droppedToolUseIds.add(targetId);
+        newBlocks.push(replaceSpeakText(targetBlock, heardText)); // partial — truncate
+        droppedSpeakIds.add(targetId); // a truncated speak's result is no longer grounded
     }
-    // Anything in this message AFTER the target block was emitted after
-    // the heard prefix and so was not heard.
+    // Blocks after the target in the same message: drop later speaks, keep the rest.
     for (let j = targetBlockIdx + 1; j < targetContent.length; j++) {
-        const id = getToolCallId(targetContent[j]);
-        if (id)
-            droppedToolUseIds.add(id);
+        const block = targetContent[j];
+        if (isSpeakToolCall(block)) {
+            const id = getToolCallId(block);
+            if (id)
+                droppedSpeakIds.add(id);
+        }
+        else {
+            newBlocks.push(block);
+        }
+    }
+    const out = messages.slice(0, targetMsgIdx);
+    if (newBlocks.length > 0) {
+        out.push({ ...targetMsg, content: newBlocks });
+    }
+    // Subsequent messages: keep them, but drop speak tool calls (and the
+    // tool_results of any dropped speak). Forward iteration guarantees a speak's
+    // id is recorded before its (later) result message is examined.
+    for (let i = targetMsgIdx + 1; i < messages.length; i++) {
+        const kept = filterPrunedSpeaks(messages[i], droppedSpeakIds);
+        if (kept)
+            out.push(kept);
+    }
+    return out;
+}
+/**
+ * Drop pruned-speak content from a trailing message: removes speak tool calls
+ * (recording their ids) and the tool_results of any dropped speak, keeping all
+ * other content. Returns the (possibly rewritten) message, or null if it ends
+ * up empty.
+ */
+function filterPrunedSpeaks(msg, droppedSpeakIds) {
+    // pi's runtime shape: a tool result is its own message (role 'toolResult')
+    // referencing one toolCallId at the message level.
+    if (isToolResultMessage(msg)) {
+        const ref = toolResultMessageRefId(msg);
+        return ref && droppedSpeakIds.has(ref) ? null : msg;
+    }
+    const content = contentOf(msg);
+    if (content.length === 0)
+        return msg;
+    let changed = false;
+    const kept = [];
+    for (const block of content) {
+        if (isSpeakToolCall(block)) {
+            const id = getToolCallId(block);
+            if (id)
+                droppedSpeakIds.add(id);
+            changed = true;
+            continue;
+        }
+        // Block-level tool_result (Anthropic shape) for a dropped speak.
+        if (isToolResultBlock(block)) {
+            const ref = toolResultBlockRefId(block);
+            if (ref && droppedSpeakIds.has(ref)) {
+                changed = true;
+                continue;
+            }
+        }
+        kept.push(block);
     }
-    const rewrittenTarget = {
-        ...targetMsg,
-        content: newBlocks,
-        stopReason: 'aborted',
-    };
-    // Anything AFTER the target message in the array was emitted by the
-    // agent after the interrupted speak — drop it. This includes any
-    // tool_result messages whose paired speak we just truncated, plus
-    // any subsequent assistant messages.
-    return [...messages.slice(0, targetMsgIdx), rewrittenTarget];
+    if (!changed)
+        return msg;
+    if (kept.length === 0)
+        return null;
+    return { ...msg, content: kept };
 }
 // ---------------------------------------------------------------------------
 // Shape-tolerant accessors. pi-agent-core's runtime AgentMessage uses
@@ -165,6 +217,22 @@ function getToolCallId(block) {
     const id = block.id;
     return typeof id === 'string' ? id : undefined;
 }
+/** pi runtime tool-result message (role 'toolResult', message-level toolCallId). */
+function isToolResultMessage(msg) {
+    return msg.role === 'toolResult';
+}
+function toolResultMessageRefId(msg) {
+    const id = msg.toolCallId;
+    return typeof id === 'string' ? id : undefined;
+}
+/** Block-level tool result (Anthropic `tool_result` / pi `toolResult`). */
+function isToolResultBlock(block) {
+    return block.type === 'tool_result' || block.type === 'toolResult';
+}
+function toolResultBlockRefId(block) {
+    const ref = block.tool_use_id ?? block.toolCallId;
+    return typeof ref === 'string' ? ref : undefined;
+}
 function getSpeakText(block) {
     // Try both shapes; whichever holds a string wins.
     const args = block.arguments;

package/server/dist/voice-orchestrator-boot.js

@@ -35,15 +35,32 @@ export function buildVoiceOrchestrator(args) {
         config,
         sessionManager,
         busResolver,
-        displaceOwner: async (sessionId, _newOwner) => {
+        displaceOwner: async (sessionId, newOwner) => {
             const slot = sessionManager.getSlot(sessionId);
-            const existingClientId = slot?.connection?.connectedClientId;
-            if (!existingClientId)
+            if (!slot)
                 return;
-            const existing = clientRegistry.get(existingClientId);
-            existing?.sendDisplacedEvent(sessionId);
+            // Notify the displaced owner (if a different client), then transfer
+            // ownership to the new caller through the SAME claim path open_session
+            // uses — sets slot.connection, subscribes, rebinds extensions, replays
+            // pending UI. Without the claim, slot.connection stayed the displaced
+            // client: events streamed to a dead socket and idle-reap never fired.
+            const existingClientId = slot.connection?.connectedClientId;
+            if (existingClientId && existingClientId !== newOwner.connectedClientId) {
+                clientRegistry.get(existingClientId)?.sendDisplacedEvent(sessionId);
+            }
+            await clientRegistry.get(newOwner.connectedClientId)?.claimSession(sessionId, slot);
         },
         isOwnedByVoiceCall: (sessionId) => orchestrator.isCallActive(sessionId),
+        notifyCallEnded: (sessionId) => {
+            // The voice extension self-deactivated (speechmux WS failed/dropped).
+            // Tell the owning client so its VoiceCallStore tears down instead of
+            // waiting for WebRTC to time out. (review finding H4)
+            const slot = sessionManager.getSlot(sessionId);
+            const ownerClientId = slot?.connection?.connectedClientId;
+            if (!ownerClientId)
+                return;
+            clientRegistry.get(ownerClientId)?.sendCallEndedEvent(sessionId, 'error');
+        },
     });
     return {
         orchestrator,

package/server/dist/voice-orchestrator.js

@@ -17,13 +17,41 @@ export class CallBindError extends Error {
 export class VoiceOrchestrator {
     opts;
     activeCalls = new Set();
+    /** Per-session unsubscribe fns for the `pimote:voice:deactivate` bus
+     *  listener installed on bind (so the server learns of extension-initiated
+     *  deactivations — speechmux drop / open failure). */
+    deactivateUnsubs = new Map();
     constructor(opts) {
         this.opts = opts;
     }
     /** Drop all active-call bookkeeping. Idempotent. Called on server shutdown. */
     async stop() {
+        for (const unsub of this.deactivateUnsubs.values()) {
+            try {
+                unsub();
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        this.deactivateUnsubs.clear();
         this.activeCalls.clear();
     }
+    /**
+     * Handle a `pimote:voice:deactivate` the voice extension emitted on its own
+     * (speechmux WS failed/dropped mid-call). Our own `endCall` removes the
+     * session from `activeCalls` *before* emitting deactivate, so the
+     * `activeCalls.has` guard distinguishes an extension-initiated deactivate
+     * from our own — preventing a feedback loop and a duplicate `call_ended`.
+     */
+    handleExtensionDeactivated(sessionId) {
+        if (!this.activeCalls.has(sessionId))
+            return;
+        this.activeCalls.delete(sessionId);
+        this.deactivateUnsubs.get(sessionId)?.();
+        this.deactivateUnsubs.delete(sessionId);
+        this.opts.notifyCallEnded?.(sessionId);
+    }
     /** Called by ws-handler for CallBindCommand. */
     async bindCall(args) {
         const slot = this.opts.busResolver.getSlot(args.sessionId);
@@ -49,13 +77,18 @@ export class VoiceOrchestrator {
         if (!bus) {
             throw new CallBindError('call_bind_failed_internal', 'Session has no EventBus');
         }
+        // Subscribe to extension-initiated deactivate BEFORE activating so a
+        // self-deactivate during/after activation is never missed. Replace any
+        // prior subscription for this session (force-rebind).
+        this.deactivateUnsubs.get(args.sessionId)?.();
+        this.deactivateUnsubs.set(args.sessionId, bus.on('pimote:voice:deactivate', () => this.handleExtensionDeactivated(args.sessionId)));
+        this.activeCalls.add(args.sessionId);
         const activate = {
             type: 'pimote:voice:activate',
             sessionId: args.sessionId,
             speechmuxWsUrl: llmWsUrl,
         };
         bus.emit(activate.type, activate);
-        this.activeCalls.add(args.sessionId);
         return {
             sessionId: args.sessionId,
             webrtcSignalUrl: signalUrl,
@@ -66,6 +99,10 @@ export class VoiceOrchestrator {
         if (!this.activeCalls.has(args.sessionId))
             return;
         this.activeCalls.delete(args.sessionId);
+        // Drop our deactivate subscription before emitting so our own emit can't
+        // loop back into handleExtensionDeactivated.
+        this.deactivateUnsubs.get(args.sessionId)?.();
+        this.deactivateUnsubs.delete(args.sessionId);
         const bus = this.opts.busResolver.getEventBus(args.sessionId);
         if (bus) {
             const deactivate = { type: 'pimote:voice:deactivate', sessionId: args.sessionId };