@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/api/responses/conversation-message.ts

@@ -33,6 +33,10 @@ import {
 import { AnsweredQuestionSchema } from "../events/question-answered.js";
 import { QuestionEntrySchema } from "../events/question-request.js";
 import { ToolActivityMetadataSchema } from "../events/tool-result.js";
+import {
+  ModeSessionActivitySchema,
+  ModeSessionSchema,
+} from "../mode-session.js";
 
 // ---------------------------------------------------------------------------
 // Attachment metadata
@@ -51,6 +55,9 @@ export const ConversationMessageAttachmentSchema = z.object({
   thumbnailData: z.string().optional(),
   /** True when the attachment bytes are backed by a file on disk. */
   fileBacked: z.boolean().optional(),
+  /** True when this computer-use screenshot was placed on the reply
+   *  automatically. Missing or false identifies legacy or explicit placement. */
+  computerUseScreenshot: z.boolean().optional(),
 });
 export type ConversationMessageAttachment = z.infer<
   typeof ConversationMessageAttachmentSchema
@@ -489,6 +496,8 @@ export type ConversationContentBlock = z.infer<
  */
 export const ConversationMessageSchema = z.object({
   id: z.string(),
+  modeSession: ModeSessionSchema.optional(),
+  modeSessionActivity: ModeSessionActivitySchema.optional(),
   /**
    * Server message ids folded into this display row when consecutive
    * assistant messages were consolidated for history rendering.

package/src/approvals/AGENTS.md

@@ -11,7 +11,9 @@ The end-to-end lifecycle of interactive guardian requests (approvals, questions)
 
 ## Single-Guardian Invariant
 
-Each assistant instance serves exactly one guardian. Multi-guardian is not supported and will never be. All connections, browser sessions, approval channels, and trust contexts within a single assistant process belong to the same guardian principal. Do not introduce guardian-keyed maps, per-guardian routing logic, or multi-guardian multiplexing — they add complexity without a real use case and create the false impression that cross-guardian isolation is required.
+Each assistant instance serves exactly one guardian. Multi-guardian is not supported and will never be. Only that guardian's principal decides guardian requests: `applyGuardianDecision()` requires it exactly. Do not introduce guardian-keyed maps, per-guardian routing logic, or multi-guardian multiplexing; they add complexity without a real use case and create the false impression that cross-guardian isolation is required.
+
+One guardian does not mean one person connected. The event stream (`GET /v1/events`) admits any actor principal with `chat.read`, so a non-guardian actor can hold a valid connection. Routes that must be guardian-only set `requireGuardian: true`, which `routes/http-adapter.ts` enforces with `requireBoundGuardian` (`runtime/auth/require-bound-guardian.ts`). Content meant only for the guardian is scoped to the guardian's principal at delivery (`targetActorPrincipalId` on the event hub), never assumed safe because it reached a connection.
 
 ## Guardian Verification Invariant
 
@@ -21,7 +23,7 @@ Conversational guardian verification control-plane invocation is guardian-only.
 
 ## Memory Provenance Invariant
 
-All memory retrieval decisions must consider actor-role provenance. Untrusted actors (non-guardian, unverified_channel) must not receive memory recall results. This invariant is enforced in `indexer.ts` (write gate) and the memory plugin's `injectors.ts` (read gate), which admits personal-memory content only for a turn whose trust context passes `isPersonalMemoryAllowed`.
+Memory access follows the acting actor's capabilities: `resolveCapabilities(trustClass).canAccessMemory`, which only `guardian` holds. An actor without it neither reads nor writes the guardian's memory. The memory plugin's `injectors.ts` admits personal-memory content only for a turn whose trust context passes `isPersonalMemoryAllowed`; the `recall`, `remember`, and `delete_memory_page` tools refuse such a turn (`plugins/defaults/memory/tools.ts`); and extraction from conversation messages runs only for guardian-provenance rows, or legacy rows with none (`plugins/defaults/memory/indexer.ts`).
 
 ## Guardian Privilege Isolation Invariant
 

package/src/approvals/approval-primitive.ts

@@ -213,7 +213,10 @@ function consumeGrantSync(params: ConsumeGrantParams): ConsumeGrantResult {
       return { ok: true, grant: reqResult.grant };
     }
 
-    log.info(
+    // Debug, not info: `consumeGrantForInvocation` and the inline grant wait
+    // poll this function, so a miss is the expected steady state while a
+    // guardian decides. Hits and the wait's outcome are logged at info.
+    log.debug(
       {
         event: "approval_primitive_consume_miss",
         mode: "request_id",
@@ -255,7 +258,7 @@ function consumeGrantSync(params: ConsumeGrantParams): ConsumeGrantResult {
     return { ok: true, grant: sigResult.grant };
   }
 
-  log.info(
+  log.debug(
     {
       event: "approval_primitive_consume_miss",
       mode: "tool_signature",

package/src/approvals/scoped-approval-grants.ts

@@ -193,7 +193,10 @@ function consumeScopedApprovalGrantByRequestId(
       .get();
 
     if (!candidate) {
-      log.info(
+      // Debug, not info: the inline grant wait and the voice retry loop call
+      // this every poll, so a miss is the expected steady state while a
+      // guardian decides. The caller logs the outcome of the wait.
+      log.debug(
         {
           event: "scoped_grant_consume_miss",
           requestId,
@@ -364,7 +367,8 @@ function consumeScopedApprovalGrantByToolSignature(
       .get();
 
     if (!candidate) {
-      log.info(
+      // Debug for the same reason as the request-id miss above: polled.
+      log.debug(
         {
           event: "scoped_grant_consume_miss",
           toolName: params.toolName,

package/src/browser/__tests__/operations.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 
 import {
   BROWSER_OPERATION_META,
+  browserOperationLifecycle,
   executeBrowserOperation,
 } from "../operations.js";
 import { BROWSER_OPERATIONS, type BrowserOperation } from "../types.js";
@@ -19,6 +20,34 @@ describe("browser operations contract", () => {
     expect(BROWSER_OPERATION_META).toHaveLength(BROWSER_OPERATIONS.length);
   });
 
+  test("classifies every typed operation for session lifecycle", () => {
+    const classifications = Object.fromEntries(
+      BROWSER_OPERATIONS.map((operation) => [
+        operation,
+        browserOperationLifecycle(operation),
+      ]),
+    );
+    expect(classifications).toEqual({
+      navigate: "action",
+      snapshot: "action",
+      screenshot: "action",
+      close: "terminal",
+      attach: "action",
+      detach: "terminal",
+      click: "action",
+      type: "action",
+      press_key: "action",
+      scroll: "action",
+      select_option: "action",
+      hover: "action",
+      wait_for: "action",
+      extract: "action",
+      wait_for_download: "action",
+      fill_credential: "action",
+      status: "status",
+    });
+  });
+
   // ── Every operation has a dispatch handler ─────────────────────────
 
   test("every operation has a dispatch handler (rejects unknown)", async () => {

package/src/browser/operation-meta.ts

@@ -216,7 +216,7 @@ Examples:
         name: "key",
         type: "string",
         description:
-          'The key to press (e.g. "Enter", "Escape", "Tab", "ArrowDown").',
+          'The key or modifier chord to press (e.g. "Enter", "Meta+v", "Control+a").',
         required: true,
       },
       {
@@ -238,6 +238,7 @@ the key is dispatched to that element; otherwise to the focused element.
 Examples:
   $ assistant browser press-key --key Enter
   $ assistant browser press-key --key Tab --element-id e5
+  $ assistant browser press-key --key "Meta+v"
   $ assistant browser press-key --key Escape`,
   },
   {

package/src/browser/operations.ts

@@ -36,6 +36,35 @@ import type { ToolExecutionResult } from "../tools/types.js";
 import type { BrowserOperationContext as ToolContext } from "./types.js";
 import type { BrowserOperation } from "./types.js";
 
+export type BrowserOperationLifecycle = "action" | "terminal" | "status";
+
+const OPERATION_LIFECYCLE: Record<BrowserOperation, BrowserOperationLifecycle> =
+  {
+    navigate: "action",
+    snapshot: "action",
+    screenshot: "action",
+    close: "terminal",
+    attach: "action",
+    detach: "terminal",
+    click: "action",
+    type: "action",
+    press_key: "action",
+    scroll: "action",
+    select_option: "action",
+    hover: "action",
+    wait_for: "action",
+    extract: "action",
+    wait_for_download: "action",
+    fill_credential: "action",
+    status: "status",
+  };
+
+export function browserOperationLifecycle(
+  operation: BrowserOperation,
+): BrowserOperationLifecycle {
+  return OPERATION_LIFECYCLE[operation];
+}
+
 // ── Dispatch handlers ────────────────────────────────────────────────
 
 /**

package/src/browser/virtual-desktop-target.ts

@@ -1,5 +1,4 @@
-import { getConfig } from "../config/loader.js";
-import { isVirtualDesktopEnabled } from "../desktop/virtual-desktop-feature.js";
+import { shouldUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
 import { browserManager } from "../tools/browser/browser-manager.js";
 import { normalizeBrowserMode } from "../tools/browser/browser-mode.js";
 import { getPinnedTab } from "../tools/browser/pinned-tabs.js";
@@ -19,16 +18,10 @@ export function shouldUseVirtualDesktopBrowser(
     mode.mode !== "auto" ||
     input.target_client_id ||
     input.use_active_tab ||
-    context.transportInterface !== "web" ||
-    context.clientOs === "macos" ||
-    context.clientOs === "windows" ||
-    context.clientOs === "linux" ||
-    context.trustClass !== "guardian" ||
-    !context.sourceActorPrincipalId ||
     browserManager.getPreferredBackendKind(context.conversationId) !== null ||
     getPinnedTab(context.conversationId)
   ) {
     return false;
   }
-  return isVirtualDesktopEnabled(getConfig());
+  return shouldUseVirtualDesktop(context);
 }

package/src/bundler/bundle-signer.ts

@@ -115,7 +115,7 @@ export async function signBundle(
   const { signature, keyId } = await requestSignature(canonicalPayload);
 
   // 5. Build SignatureJson
-  const signatureJson: SignatureJson = {
+  return {
     algorithm: "ed25519",
     signer: {
       key_id: keyId,
@@ -124,6 +124,4 @@ export async function signBundle(
     content_hashes: contentHashes,
     signature,
   };
-
-  return signatureJson;
 }

package/src/calls/__tests__/barge-in-guard.test.ts

@@ -12,13 +12,16 @@ describe("createBargeInGuard", () => {
     expect(guard.track("speech", 100)).toBe("pending");
     expect(guard.track("speech", 100)).toBe("pending");
     expect(guard.speechMs).toBe(200);
+    expect(guard.fired).toBe(false);
   });
 
   test("sustained speech reaching the threshold fires once and stays fired", () => {
     const guard = createBargeInGuard(250);
     guard.track("speech", 200);
     expect(guard.track("speech", 50)).toBe("fired");
+    expect(guard.fired).toBe(true);
     expect(guard.track("silence", 1_000)).toBe("fired");
+    expect(guard.fired).toBe(true);
   });
 
   test("a brief sub-threshold gap does not reset the run", () => {

package/src/calls/__tests__/call-controller.test.ts

@@ -421,6 +421,7 @@ function setupController(
 ) {
   ensureConversation("conv-ctrl-test");
   const session = createCallSession({
+    direction: "inbound",
     conversationId: "conv-ctrl-test",
     provider: "twilio",
     fromNumber: "+15551111111",
@@ -471,6 +472,7 @@ function setupControllerWithOrigin(task?: string) {
   ensureConversation("conv-ctrl-voice");
   ensureConversation("conv-ctrl-origin");
   const session = createCallSession({
+    direction: "inbound",
     conversationId: "conv-ctrl-voice",
     provider: "twilio",
     fromNumber: "+15551111111",
@@ -5028,4 +5030,96 @@ describe("call-controller", () => {
       await turnPromise.catch(() => {});
     });
   });
+  describe("turn metrics", () => {
+    test("a caller turn records the marks the latency numbers are built from", async () => {
+      const { controller } = setupController();
+
+      await controller.handleCallerUtterance("What is the weather?");
+
+      const snapshot = controller.getMetricsSnapshot();
+      expect(snapshot.recentTurns.length).toBe(1);
+      const turn = snapshot.recentTurns[0];
+      expect(turn.status).toBe("completed");
+      // The transcript mark is seeded from when the caller stopped talking,
+      // the dispatch mark from when the leg went out, and the delta mark from
+      // the first token back: the three anchors every phone latency number is
+      // subtracted from.
+      expect(turn.timestamps.finalTranscriptAtMs).not.toBeNull();
+      expect(turn.timestamps.assistantDispatchAtMs).not.toBeNull();
+      expect(turn.timestamps.firstAssistantDeltaAtMs).not.toBeNull();
+      expect(turn.durations.dispatchToFirstAssistantDeltaMs).not.toBeNull();
+      expect(turn.durations.totalTurnDurationMs).not.toBeNull();
+      // The provider's final doubles as the speech-end anchor, so the turn's
+      // round trip is measurable on phone as it is in live voice.
+      expect(turn.timestamps.utteranceEndAtMs).not.toBeNull();
+      expect(turn.durations.roundTripMs).not.toBeNull();
+
+      controller.destroy();
+    });
+
+    test("the transcript mark predates the dispatch it triggered", async () => {
+      // Seeding backdates the turn to the caller's own boundary; a turn that
+      // opened at dispatch time would report a felt latency that starts after
+      // the caller already finished speaking.
+      const { controller } = setupController();
+
+      await controller.handleCallerUtterance("Hello there");
+
+      const turn = controller.getMetricsSnapshot().recentTurns[0];
+      const finalTranscriptAtMs = turn.timestamps.finalTranscriptAtMs;
+      const assistantDispatchAtMs = turn.timestamps.assistantDispatchAtMs;
+      expect(finalTranscriptAtMs).not.toBeNull();
+      expect(assistantDispatchAtMs).not.toBeNull();
+      expect(finalTranscriptAtMs!).toBeLessThanOrEqual(assistantDispatchAtMs!);
+      expect(turn.timestamps.startedAtMs).toBeLessThanOrEqual(
+        finalTranscriptAtMs!,
+      );
+
+      controller.destroy();
+    });
+
+    test("a barge-in ends the turn once, as cancelled", async () => {
+      // One interruption is one cancelled turn: the count downstream reads as
+      // "turns the caller cut off", so a second settle would double it.
+      // A turn that never emits or completes holds the controller in the
+      // pre-speech phase, which a sustained barge-in can still cut off.
+      mockStartVoiceTurn.mockImplementation(
+        async (opts: { onComplete: () => void }) => ({
+          turnId: "run-metrics-slow",
+          abort: () => opts.onComplete(),
+        }),
+      );
+      const { controller } = setupController();
+      const turnPromise = controller.handleCallerUtterance("Tell me a story");
+      for (let i = 0; i < 5; i++) {
+        await Promise.resolve();
+      }
+      expect(controller.getState()).toBe("processing");
+
+      expect(controller.handleBargeIn()).toBe(true);
+      await turnPromise.catch(() => {});
+      controller.destroy();
+
+      const turns = controller.getMetricsSnapshot().recentTurns;
+      expect(turns.length).toBe(1);
+      expect(turns[0].status).toBe("cancelled");
+      expect(turns[0].cancellationReason).toBe("interrupted");
+      expect(turns[0].timestamps.bargeInAtMs).not.toBeNull();
+    });
+
+    test("no turn is invented between turns", async () => {
+      // Every mark opens a turn when none is active, so a barge-in on an idle
+      // controller must leave the collector empty rather than start a turn
+      // that never settles.
+      const { controller } = setupController();
+
+      expect(controller.handleBargeIn()).toBe(false);
+
+      const snapshot = controller.getMetricsSnapshot();
+      expect(snapshot.recentTurns.length).toBe(0);
+      expect(snapshot.activeTurn).toBeNull();
+
+      controller.destroy();
+    });
+  });
 });

package/src/calls/__tests__/call-funnel-telemetry.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Phone-call funnel rows: which ones the call store records and what they say.
+ *
+ * The recorder is mocked, as in `live-voice-session-telemetry.test.ts`: the
+ * outbox and its consent gate are covered by `telemetry-events-outbox.test.ts`.
+ * What these tests pin is the wiring that decides whether a row exists at all,
+ * which is where the measurement silently goes wrong: a call that never
+ * connects still has to count, a terminal status has to emit exactly one end
+ * row however many writes follow it, and a call nobody spoke on has to say so.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+const recordPhoneCallStarted = mock(() => null);
+const recordPhoneCallEnded = mock(() => null);
+
+// Spread the real module rather than replacing it: the call store pulls other
+// recorders out of it, and a bare replacement makes those imports fail.
+const onboardingEventsStore =
+  await import("../../onboarding/onboarding-events-store.js");
+mock.module("../../onboarding/onboarding-events-store.js", () => ({
+  ...onboardingEventsStore,
+  recordPhoneCallStarted,
+  recordPhoneCallEnded,
+}));
+
+import { getDb } from "../../persistence/db-connection.js";
+import { initializeDb } from "../../persistence/db-init.js";
+import { conversations } from "../../persistence/schema/index.js";
+
+await initializeDb();
+
+// Values load after `mock.module` so the store picks up the mocked recorder.
+const { createCallSession, recordCallEvent, updateCallSession } =
+  await import("../call-store.js");
+
+let conversationSeq = 0;
+
+function makeConversation(): string {
+  const id = `conv-funnel-${++conversationSeq}`;
+  const now = Date.now();
+  getDb()
+    .insert(conversations)
+    .values({
+      id,
+      title: `Test conversation ${id}`,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .run();
+  return id;
+}
+
+function startCall(opts?: {
+  direction?: "inbound" | "outbound";
+  callMode?: string;
+}): string {
+  const session = createCallSession({
+    conversationId: makeConversation(),
+    provider: "twilio",
+    direction: opts?.direction ?? "inbound",
+    fromNumber: "+15550100",
+    toNumber: "+15550101",
+    ...(opts?.callMode !== undefined ? { callMode: opts.callMode } : {}),
+  });
+  return session.id;
+}
+
+beforeEach(() => {
+  recordPhoneCallStarted.mockClear();
+  recordPhoneCallEnded.mockClear();
+});
+
+describe("call funnel start rows", () => {
+  test("an inbound call counts the moment its session row exists", () => {
+    // Before the provider has dialled and before any preflight can reject it:
+    // a call that never connects is the one the funnel most needs to count.
+    const callSessionId = startCall({ callMode: "normal" });
+
+    expect(recordPhoneCallStarted).toHaveBeenCalledTimes(1);
+    expect(recordPhoneCallStarted).toHaveBeenCalledWith({
+      callSessionId,
+      screen: "started_inbound:normal",
+    });
+  });
+
+  test("an outbound call is stamped by the direction it states", () => {
+    // Stated, not inferred: a verification or invite call dials out and
+    // carries no task, so task presence would misread as inbound.
+    startCall({ direction: "outbound", callMode: "verification" });
+
+    expect(recordPhoneCallStarted).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "started_outbound:verification" }),
+    );
+  });
+
+  test("an ordinary call stamps the normal mode it never writes", () => {
+    // createInboundVoiceSession and startCall leave callMode null; only the
+    // verification and invite flows set one.
+    startCall();
+
+    expect(recordPhoneCallStarted).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "started_inbound:normal" }),
+    );
+  });
+});
+
+describe("call funnel end rows", () => {
+  test("a completed call that took a turn reports no silence", () => {
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "in_progress" });
+    recordCallEvent(callSessionId, "call_connected");
+    recordCallEvent(callSessionId, "caller_spoke", {
+      transcript: "hello",
+      transport: "media-stream",
+    });
+
+    updateCallSession(callSessionId, { status: "completed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledTimes(1);
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith({
+      callSessionId,
+      screen: "ended_completed",
+      outcome: "completed",
+    });
+  });
+
+  test("a call that never connected is silent for that reason", () => {
+    const callSessionId = startCall({ callMode: "normal" });
+
+    updateCallSession(callSessionId, { status: "failed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith({
+      callSessionId,
+      screen: "ended_failed:silent_no_connect",
+      outcome: "failed",
+    });
+  });
+
+  test("a connected call nobody spoke on is silent on the line", () => {
+    // The distinction the rate lives on: the telephony leg worked and the
+    // conversation still never started.
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "in_progress" });
+    recordCallEvent(callSessionId, "call_connected");
+
+    updateCallSession(callSessionId, { status: "completed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "ended_completed:silent_no_turn" }),
+    );
+  });
+
+  test("a call where only digits were pressed took no turn", () => {
+    // DTMF digits ride `caller_spoke` too; only a transcript is a turn.
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "in_progress" });
+    recordCallEvent(callSessionId, "call_connected");
+    recordCallEvent(callSessionId, "caller_spoke", {
+      dtmfDigit: "4",
+      transport: "media-stream",
+    });
+
+    updateCallSession(callSessionId, { status: "completed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "ended_completed:silent_no_turn" }),
+    );
+  });
+
+  test("a cancelled call completed its course", () => {
+    // Only an error is a failure; a call cancelled before it connected ran
+    // exactly as far as it was asked to.
+    const callSessionId = startCall({ callMode: "normal" });
+
+    updateCallSession(callSessionId, { status: "cancelled" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ outcome: "completed" }),
+    );
+  });
+
+  test("only the first terminal write records an end", () => {
+    // Terminal states are immutable, so the rejected second write must not
+    // produce a second row to pair with the one start.
+    const callSessionId = startCall({ callMode: "normal" });
+    updateCallSession(callSessionId, { status: "completed" });
+
+    updateCallSession(callSessionId, { status: "failed" });
+
+    expect(recordPhoneCallEnded).toHaveBeenCalledTimes(1);
+    expect(recordPhoneCallEnded).toHaveBeenCalledWith(
+      expect.objectContaining({ screen: "ended_completed:silent_no_connect" }),
+    );
+  });
+
+  test("a non-status update records nothing", () => {
+    const callSessionId = startCall({ callMode: "normal" });
+
+    updateCallSession(callSessionId, { providerCallSid: "CA-test" });
+
+    expect(recordPhoneCallEnded).not.toHaveBeenCalled();
+  });
+});

package/src/calls/__tests__/voice-control-protocol.test.ts

@@ -10,6 +10,7 @@ import {
   type SessionControlRequest,
   stripInternalSpeechMarkers,
   TASK_STOP_MARKER,
+  TASK_UPDATE_SILENT_MARKER,
   terminalControlMarkerLength,
 } from "../voice-control-protocol.js";
 
@@ -227,3 +228,17 @@ describe("session control markers", () => {
     expect(terminalControlMarkerLength("The array [-1] sorts")).toBe(0);
   });
 });
+
+test("silent task-update marker is held across every split and removed from the transcript", () => {
+  for (let split = 1; split < TASK_UPDATE_SILENT_MARKER.length; split += 1) {
+    const emitted: string[] = [];
+    const flush = createControlMarkerHoldback((text) => emitted.push(text));
+    flush(TASK_UPDATE_SILENT_MARKER.slice(0, split));
+    flush(TASK_UPDATE_SILENT_MARKER, { force: true });
+    expect(emitted.join("")).toBe("");
+  }
+  expect(terminalControlMarkerLength(TASK_UPDATE_SILENT_MARKER)).toBe(
+    TASK_UPDATE_SILENT_MARKER.length,
+  );
+  expect(parseTerminalSessionControl(TASK_UPDATE_SILENT_MARKER)).toBeNull();
+});