@vellumai/assistant 0.3.13 → 0.3.15

package/src/daemon/handlers/sessions.ts

@@ -35,10 +35,12 @@ import type {
   UserMessage,
 } from '../ipc-protocol.js';
 import { normalizeThreadType } from '../ipc-protocol.js';
-import { classifyRecordingIntent, detectRecordingIntent, detectStopRecordingIntent, hasSubstantiveContent, isInterrogative, stripRecordingIntent, stripStopRecordingIntent } from '../recording-intent.js';
+import { executeRecordingIntent } from '../recording-executor.js';
+import { resolveRecordingIntent } from '../recording-intent.js';
+import { classifyRecordingIntentFallback, containsRecordingKeywords } from '../recording-intent-fallback.js';
 import { buildSessionErrorMessage,classifySessionError } from '../session-error.js';
 import { generateVideoThumbnail } from '../video-thumbnail.js';
-import { handleRecordingStart, handleRecordingStop } from './recording.js';
+import { handleRecordingPause, handleRecordingRestart, handleRecordingResume, handleRecordingStart, handleRecordingStop } from './recording.js';
 import {
   defineHandlers,
   type HandlerContext,
@@ -175,7 +177,7 @@ export async function handleUserMessage(
     };
 
     const config = getConfig();
-    const messageText = msg.content ?? '';
+    let messageText = msg.content ?? '';
 
     // Block inbound messages that contain secrets and redirect to secure prompt
     if (!msg.bypassSecretCheck) {
@@ -227,85 +229,157 @@ export async function handleUserMessage(
       }
     }
 
+    // ── Structured command intent (bypasses text parsing) ──────────────────
+    if (config.daemon.standaloneRecording && msg.commandIntent?.domain === 'screen_recording') {
+      const action = msg.commandIntent.action;
+      rlog.info({ action, source: 'commandIntent' }, 'Recording command intent received in user_message');
+      if (action === 'start') {
+        const recordingId = handleRecordingStart(msg.sessionId, { promptForSource: true }, socket, ctx);
+        ctx.send(socket, {
+          type: 'assistant_text_delta',
+          text: recordingId ? 'Starting screen recording.' : 'A recording is already active.',
+          sessionId: msg.sessionId,
+        });
+        ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
+        return;
+      } else if (action === 'stop') {
+        const stopped = handleRecordingStop(msg.sessionId, ctx) !== undefined;
+        ctx.send(socket, {
+          type: 'assistant_text_delta',
+          text: stopped ? 'Stopping the recording.' : 'No active recording to stop.',
+          sessionId: msg.sessionId,
+        });
+        ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
+        return;
+      } else if (action === 'restart') {
+        const restartResult = handleRecordingRestart(msg.sessionId, socket, ctx);
+        ctx.send(socket, {
+          type: 'assistant_text_delta',
+          text: restartResult.responseText,
+          sessionId: msg.sessionId,
+        });
+        ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
+        return;
+      } else if (action === 'pause') {
+        const paused = handleRecordingPause(msg.sessionId, ctx) !== undefined;
+        ctx.send(socket, {
+          type: 'assistant_text_delta',
+          text: paused ? 'Pausing the recording.' : 'No active recording to pause.',
+          sessionId: msg.sessionId,
+        });
+        ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
+        return;
+      } else if (action === 'resume') {
+        const resumed = handleRecordingResume(msg.sessionId, ctx) !== undefined;
+        ctx.send(socket, {
+          type: 'assistant_text_delta',
+          text: resumed ? 'Resuming the recording.' : 'No active recording to resume.',
+          sessionId: msg.sessionId,
+        });
+        ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
+        return;
+      } else {
+        // Unrecognized action — fall through to normal text handling
+        rlog.warn({ action, source: 'commandIntent' }, 'Unrecognized screen_recording action, falling through to text handling');
+      }
+    }
+
     // ── Standalone recording intent interception ──────────────────────────
     if (config.daemon.standaloneRecording && messageText) {
       const name = getAssistantName();
       const dynamicNames = [name].filter(Boolean) as string[];
-      const intentClass = classifyRecordingIntent(messageText, dynamicNames);
+      const intentResult = resolveRecordingIntent(messageText, dynamicNames);
+
+      if (intentResult.kind === 'start_only' || intentResult.kind === 'stop_only' ||
+          intentResult.kind === 'start_and_stop_only' ||
+          intentResult.kind === 'restart_only' || intentResult.kind === 'pause_only' ||
+          intentResult.kind === 'resume_only') {
+        const execResult = executeRecordingIntent(intentResult, {
+          conversationId: msg.sessionId,
+          socket,
+          ctx,
+        });
 
-      switch (intentClass) {
-        case 'stop_only': {
-          const stopped = handleRecordingStop(msg.sessionId, ctx) !== undefined;
-          rlog.info('Recording stop intent intercepted in user_message');
+        if (execResult.handled) {
+          rlog.info({ kind: intentResult.kind }, 'Recording intent intercepted in user_message');
           ctx.send(socket, {
             type: 'assistant_text_delta',
-            text: stopped ? 'Stopping the recording.' : 'No active recording to stop.',
+            text: execResult.responseText!,
             sessionId: msg.sessionId,
           });
           ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
           return;
         }
-        case 'start_only': {
-          const recordingId = handleRecordingStart(msg.sessionId, { promptForSource: true }, socket, ctx);
-          rlog.info('Recording-only intent intercepted in user_message');
-
-          if (recordingId) {
-            ctx.send(socket, { type: 'assistant_text_delta', text: 'Starting screen recording.', sessionId: msg.sessionId });
-          } else {
-            ctx.send(socket, { type: 'assistant_text_delta', text: 'A recording is already active.', sessionId: msg.sessionId });
-          }
-          ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
-          return;
+      }
+
+      if (intentResult.kind === 'start_with_remainder' || intentResult.kind === 'stop_with_remainder' ||
+          intentResult.kind === 'start_and_stop_with_remainder' || intentResult.kind === 'restart_with_remainder') {
+        const execResult = executeRecordingIntent(intentResult, {
+          conversationId: msg.sessionId,
+          socket,
+          ctx,
+        });
+
+        // Continue with stripped text for downstream processing
+        msg.content = execResult.remainderText ?? messageText;
+        messageText = msg.content;
+
+        // Execute the recording side effects that executeRecordingIntent deferred
+        if (intentResult.kind === 'stop_with_remainder') {
+          handleRecordingStop(msg.sessionId, ctx);
+        }
+        if (intentResult.kind === 'start_with_remainder') {
+          handleRecordingStart(msg.sessionId, { promptForSource: true }, socket, ctx);
         }
-        case 'mixed': {
-          // Skip recording side effects for questions about recording
-          // (e.g., "how do I stop recording?") — let the model answer instead.
-          if (isInterrogative(messageText, dynamicNames)) {
-            rlog.info('Mixed recording intent is interrogative — skipping side effects');
-            break;
+        // start_and_stop_with_remainder / restart_with_remainder — route through
+        // handleRecordingRestart which properly cleans up maps between stop and start.
+        if (intentResult.kind === 'restart_with_remainder' || intentResult.kind === 'start_and_stop_with_remainder') {
+          const restartResult = handleRecordingRestart(msg.sessionId, socket, ctx);
+          // Only fall back to plain start for start_and_stop_with_remainder.
+          // restart_with_remainder should NOT silently start a new recording when idle.
+          if (!restartResult.initiated && restartResult.reason === 'no_active_recording'
+              && intentResult.kind === 'start_and_stop_with_remainder') {
+            handleRecordingStart(msg.sessionId, { promptForSource: true }, socket, ctx);
           }
+        }
 
-          // Mixed = recording intent embedded in broader text.
-          // Handle the recording action, then check if remaining text is substantive.
-          const hasStart = detectRecordingIntent(messageText);
-          const hasStop = detectStopRecordingIntent(messageText);
+        rlog.info({ remaining: msg.content, kind: intentResult.kind }, 'Recording intent with remainder — continuing with remaining text');
+      }
 
-          if (hasStop) {
-            handleRecordingStop(msg.sessionId, ctx);
-            rlog.info('Mixed intent — stopping recording');
-          }
-          const startResult = hasStart ? handleRecordingStart(msg.sessionId, { promptForSource: true }, socket, ctx) : null;
-          if (hasStart) {
-            rlog.info({ started: !!startResult }, 'Mixed intent — starting recording');
-          }
+      // 'none' — deterministic resolver found nothing; try LLM fallback
+      // if the text contains recording-related keywords.
+      if (intentResult.kind === 'none' && containsRecordingKeywords(messageText)) {
+        const fallback = await classifyRecordingIntentFallback(messageText);
+        rlog.info({ fallbackAction: fallback.action, fallbackConfidence: fallback.confidence }, 'Recording intent LLM fallback result');
+
+        if (fallback.action !== 'none' && fallback.confidence === 'high') {
+          const kindMap: Record<string, import('../recording-intent.js').RecordingIntentResult> = {
+            start: { kind: 'start_only' },
+            stop: { kind: 'stop_only' },
+            restart: { kind: 'restart_only' },
+            pause: { kind: 'pause_only' },
+            resume: { kind: 'resume_only' },
+          };
+          const mapped = kindMap[fallback.action];
+          if (mapped) {
+            const execResult = executeRecordingIntent(mapped, {
+              conversationId: msg.sessionId,
+              socket,
+              ctx,
+            });
 
-          // Strip recording clauses from the message
-          let remaining = messageText;
-          if (hasStart) remaining = stripRecordingIntent(remaining);
-          if (hasStop) remaining = stripStopRecordingIntent(remaining);
-
-          // If nothing substantive remains (just fillers, names, punctuation), complete now
-          if (!hasSubstantiveContent(remaining, dynamicNames)) {
-            let text: string;
-            if (hasStart && startResult) {
-              text = hasStop ? 'Stopping current recording and starting a new one.' : 'Starting screen recording.';
-            } else if (hasStart) {
-              text = 'A recording is already active.';
-            } else {
-              text = 'Stopping the recording.';
+            if (execResult.handled) {
+              rlog.info({ kind: mapped.kind, source: 'llm_fallback' }, 'Recording intent intercepted via LLM fallback');
+              ctx.send(socket, {
+                type: 'assistant_text_delta',
+                text: execResult.responseText!,
+                sessionId: msg.sessionId,
+              });
+              ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
+              return;
             }
-            ctx.send(socket, { type: 'assistant_text_delta', text, sessionId: msg.sessionId });
-            ctx.send(socket, { type: 'message_complete', sessionId: msg.sessionId });
-            return;
           }
-
-          // Continue with stripped text for downstream processing
-          msg.content = remaining;
-          rlog.info({ remaining }, 'Mixed recording intent — recording handled, continuing with remaining text');
-          break;
         }
-        case 'none':
-          break;
       }
     }
 
@@ -405,12 +479,12 @@ export function handleSessionList(socket: net.Socket, ctx: HandlerContext, offse
       const originInterface = parseInterfaceId(c.originInterface);
       const attn = attentionStates.get(c.id);
       const assistantAttention = attn ? {
-        hasUnseenLatestAssistantMessage: attn.latestAssistantMessageAt !== null &&
-          (attn.lastSeenAssistantMessageAt === null || attn.lastSeenAssistantMessageAt < attn.latestAssistantMessageAt),
-        ...(attn.latestAssistantMessageAt !== null ? { latestAssistantMessageAt: attn.latestAssistantMessageAt } : {}),
-        ...(attn.lastSeenAssistantMessageAt !== null ? { lastSeenAssistantMessageAt: attn.lastSeenAssistantMessageAt } : {}),
-        ...(attn.lastSeenConfidence !== null ? { lastSeenConfidence: attn.lastSeenConfidence } : {}),
-        ...(attn.lastSeenSignalType !== null ? { lastSeenSignalType: attn.lastSeenSignalType } : {}),
+        hasUnseenLatestAssistantMessage: attn.latestAssistantMessageAt != null &&
+          (attn.lastSeenAssistantMessageAt == null || attn.lastSeenAssistantMessageAt < attn.latestAssistantMessageAt),
+        ...(attn.latestAssistantMessageAt != null ? { latestAssistantMessageAt: attn.latestAssistantMessageAt } : {}),
+        ...(attn.lastSeenAssistantMessageAt != null ? { lastSeenAssistantMessageAt: attn.lastSeenAssistantMessageAt } : {}),
+        ...(attn.lastSeenConfidence != null ? { lastSeenConfidence: attn.lastSeenConfidence } : {}),
+        ...(attn.lastSeenSignalType != null ? { lastSeenSignalType: attn.lastSeenSignalType } : {}),
       } : undefined;
       return {
         id: c.id,

package/src/daemon/ipc-contract/computer-use.ts

@@ -1,6 +1,6 @@
 // Computer use, task routing, ride shotgun, and watch observation types.
 
-import type { IpcBlobRef,UserMessageAttachment } from './shared.js';
+import type { CommandIntent, IpcBlobRef,UserMessageAttachment } from './shared.js';
 
 // === Client → Server ===
 
@@ -53,6 +53,8 @@ export interface TaskSubmit {
   screenHeight: number;
   attachments?: UserMessageAttachment[];
   source?: 'voice' | 'text';
+  /** Structured command intent — bypasses text parsing when present. */
+  commandIntent?: CommandIntent;
 }
 
 export interface RideShotgunStart {
@@ -100,11 +102,13 @@ export interface RecordingOptions {
 export interface RecordingStatus {
   type: 'recording_status';
   sessionId: string;          // matches recordingId from RecordingStart
-  status: 'started' | 'stopped' | 'failed';
+  status: 'started' | 'stopped' | 'failed' | 'restart_cancelled' | 'paused' | 'resumed';
   filePath?: string;          // on stop
   durationMs?: number;        // on stop
   error?: string;             // on failure
   attachToConversationId?: string;
+  /** Operation token for restart race hardening — matches the token from RecordingStart. */
+  operationToken?: string;
 }
 
 // === Server → Client ===
@@ -115,6 +119,8 @@ export interface RecordingStart {
   recordingId: string;        // daemon-assigned UUID
   attachToConversationId?: string;
   options?: RecordingOptions;
+  /** Operation token for restart race hardening — stale completions with mismatched tokens are rejected. */
+  operationToken?: string;
 }
 
 /** Server → Client: stop a recording. */
@@ -123,6 +129,18 @@ export interface RecordingStop {
   recordingId: string;        // matches RecordingStart.recordingId
 }
 
+/** Server → Client: pause the active recording. */
+export interface RecordingPause {
+  type: 'recording_pause';
+  recordingId: string;
+}
+
+/** Server → Client: resume a paused recording. */
+export interface RecordingResume {
+  type: 'recording_resume';
+  recordingId: string;
+}
+
 export interface CuAction {
   type: 'cu_action';
   sessionId: string;
@@ -211,4 +229,6 @@ export type _ComputerUseServerMessages =
   | WatchStarted
   | WatchCompleteRequest
   | RecordingStart
-  | RecordingStop;
+  | RecordingStop
+  | RecordingPause
+  | RecordingResume;

package/src/daemon/ipc-contract/messages.ts

@@ -1,7 +1,7 @@
 // User/assistant messages, tool results, confirmations, secrets, errors, and generation lifecycle.
 
 import type { ChannelId, InterfaceId } from '../../channels/types.js';
-import type { UserMessageAttachment } from './shared.js';
+import type { CommandIntent, UserMessageAttachment } from './shared.js';
 
 // === Client → Server ===
 
@@ -19,6 +19,8 @@ export interface UserMessage {
   channel?: ChannelId;
   /** Originating interface identifier (e.g. 'macos'). */
   interface: InterfaceId;
+  /** Structured command intent — bypasses text parsing when present. */
+  commandIntent?: CommandIntent;
 }
 
 export interface ConfirmationResponse {

package/src/daemon/ipc-contract/shared.ts

@@ -29,6 +29,12 @@ export interface DictationContext {
   cursorInTextField: boolean;
 }
 
+/** Structured command intent — bypasses text parsing when present. */
+export interface CommandIntent {
+  domain: 'screen_recording';
+  action: 'start' | 'stop' | 'restart' | 'pause' | 'resume';
+}
+
 export interface UserMessageAttachment {
   id?: string;
   filename: string;

package/src/daemon/ipc-contract-inventory.json

@@ -273,6 +273,8 @@
     "platform_config_response",
     "pong",
     "publish_page_response",
+    "recording_pause",
+    "recording_resume",
     "recording_start",
     "recording_stop",
     "reminders_list_response",

package/src/daemon/lifecycle.ts

@@ -185,6 +185,8 @@ export async function runDaemon(): Promise<void> {
           label: reminder.label,
           message: reminder.message,
         },
+        routingIntent: reminder.routingIntent,
+        routingHints: reminder.routingHints,
         dedupeKey: `reminder:${reminder.id}`,
       });
     },

package/src/daemon/recording-executor.ts

@@ -0,0 +1,180 @@
+// Unified recording intent executor.
+// Bridges the gap between recording-intent.ts (classification) and
+// handlers/recording.ts (side effects), so both sessions.ts and misc.ts
+// can share the same execution logic without duplicating switch/case blocks.
+
+import type * as net from 'node:net';
+
+import {
+  handleRecordingPause,
+  handleRecordingRestart,
+  handleRecordingResume,
+  handleRecordingStart,
+  handleRecordingStop,
+  isRecordingIdle,
+} from './handlers/recording.js';
+import type { HandlerContext } from './handlers/shared.js';
+import type { RecordingIntentResult } from './recording-intent.js';
+
+export interface RecordingExecutionContext {
+  conversationId: string;
+  socket: net.Socket;
+  ctx: HandlerContext;
+}
+
+export interface RecordingExecutionOutput {
+  /** If true, the intent was fully handled (start_only / stop_only) -- handler should send completion and return */
+  handled: boolean;
+  /** Human-readable response text for the user */
+  responseText?: string;
+  /** For _with_remainder: the remaining text after stripping recording clauses */
+  remainderText?: string;
+  /** Whether a recording start should be/was initiated */
+  pendingStart?: boolean;
+  /** Whether a recording stop should be/was initiated */
+  pendingStop?: boolean;
+  /** Whether a restart is pending (for restart_with_remainder) */
+  pendingRestart?: boolean;
+  /** Whether handleRecordingStart succeeded (true) or was rejected (false). Only set for start_only / start_and_stop_only. */
+  recordingStarted?: boolean;
+}
+
+export function executeRecordingIntent(
+  result: RecordingIntentResult,
+  context: RecordingExecutionContext,
+): RecordingExecutionOutput {
+  switch (result.kind) {
+    case 'none':
+      return { handled: false };
+
+    case 'start_only': {
+      const recordingId = handleRecordingStart(
+        context.conversationId,
+        { promptForSource: true },
+        context.socket,
+        context.ctx,
+      );
+      return {
+        handled: true,
+        recordingStarted: !!recordingId,
+        responseText: recordingId
+          ? 'Starting screen recording.'
+          : 'A recording is already active.',
+      };
+    }
+
+    case 'stop_only': {
+      const stopped = handleRecordingStop(context.conversationId, context.ctx) !== undefined;
+      return {
+        handled: true,
+        responseText: stopped
+          ? 'Stopping the recording.'
+          : 'No active recording to stop.',
+      };
+    }
+
+    case 'start_with_remainder':
+      return {
+        handled: false,
+        remainderText: result.remainder,
+        pendingStart: true,
+      };
+
+    case 'stop_with_remainder':
+      return {
+        handled: false,
+        remainderText: result.remainder,
+        pendingStop: true,
+      };
+
+    case 'start_and_stop_only': {
+      // Route through handleRecordingRestart which properly cleans up maps
+      // between stop and start, preventing the "already active" guard from
+      // blocking the new recording.
+      const restartResult = handleRecordingRestart(
+        context.conversationId,
+        context.socket,
+        context.ctx,
+      );
+
+      // When there was no active recording to restart, fall back to a plain
+      // start — the user said "stop and start" but nothing was recording, so
+      // the stop is a no-op and we just start a new recording.
+      // Only fall back for this specific reason; "restart_in_progress" should
+      // not start a duplicate recording.
+      if (!restartResult.initiated && restartResult.reason === 'no_active_recording') {
+        const recordingId = handleRecordingStart(
+          context.conversationId,
+          { promptForSource: true },
+          context.socket,
+          context.ctx,
+        );
+        return {
+          handled: true,
+          recordingStarted: !!recordingId,
+          responseText: recordingId
+            ? 'Starting screen recording.'
+            : 'A recording is already active.',
+        };
+      }
+
+      return {
+        handled: true,
+        recordingStarted: restartResult.initiated,
+        responseText: restartResult.initiated
+          ? 'Stopping current recording and starting a new one.'
+          : restartResult.responseText,
+      };
+    }
+
+    case 'start_and_stop_with_remainder':
+      // When there's no active recording, fall back to a plain start rather
+      // than a restart — the stop is a no-op and we just need to start.
+      return {
+        handled: false,
+        remainderText: result.remainder,
+        ...(isRecordingIdle()
+          ? { pendingStart: true }
+          : { pendingRestart: true }),
+      };
+
+    case 'restart_only': {
+      const restartResult = handleRecordingRestart(
+        context.conversationId,
+        context.socket,
+        context.ctx,
+      );
+      return {
+        handled: true,
+        responseText: restartResult.responseText,
+      };
+    }
+
+    case 'restart_with_remainder':
+      return {
+        handled: false,
+        remainderText: result.remainder,
+        pendingRestart: true,
+      };
+
+    case 'pause_only': {
+      const paused = handleRecordingPause(context.conversationId, context.ctx) !== undefined;
+      return {
+        handled: true,
+        responseText: paused
+          ? 'Pausing the recording.'
+          : 'No active recording to pause.',
+      };
+    }
+
+    case 'resume_only': {
+      const resumed = handleRecordingResume(context.conversationId, context.ctx) !== undefined;
+      return {
+        handled: true,
+        responseText: resumed
+          ? 'Resuming the recording.'
+          : 'No active recording to resume.',
+      };
+    }
+  }
+}