@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/api/mode-session.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  ModeSessionSchema,
+  ModeSessionStatusSchema,
+  ModeSessionSummarySchema,
+} from "./mode-session.js";
+
+describe("mode session API schemas", () => {
+  test("accepts canonical membership", () => {
+    expect(
+      ModeSessionSchema.parse({ id: "session-123", mode: "browser" }),
+    ).toEqual({ id: "session-123", mode: "browser" });
+  });
+
+  test("keeps persisted status to the three lifecycle states", () => {
+    expect(ModeSessionStatusSchema.options).toEqual([
+      "active",
+      "completed",
+      "interrupted",
+    ]);
+    expect(ModeSessionStatusSchema.safeParse("waiting").success).toBe(false);
+    expect(ModeSessionStatusSchema.safeParse("draining").success).toBe(false);
+  });
+
+  test("accepts an interrupted summary with unknown end time", () => {
+    const summary = ModeSessionSummarySchema.parse({
+      id: "session-123",
+      conversationId: "conv-123",
+      mode: "computer_use",
+      status: "interrupted",
+      sourceStartedAt: 100,
+      firstIncludedAt: 110,
+      firstIncludedMessageId: "message-123",
+      lastActivityAt: 125,
+      lastOwnedMessageId: "message-456",
+      endedAt: null,
+      endReason: "assistant_restarted",
+      revision: 4,
+    });
+
+    expect(summary.endedAt).toBeNull();
+  });
+
+  test("rejects unknown historic modes and invalid revisions", () => {
+    const base = {
+      id: "session-123",
+      conversationId: "conv-123",
+      status: "active",
+      sourceStartedAt: 100,
+      firstIncludedAt: null,
+      firstIncludedMessageId: null,
+      lastActivityAt: 100,
+      lastOwnedMessageId: null,
+      endedAt: null,
+      endReason: null,
+    };
+    expect(
+      ModeSessionSummarySchema.safeParse({
+        ...base,
+        mode: "unknown_mode",
+        revision: 1,
+      }).success,
+    ).toBe(false);
+    expect(
+      ModeSessionSummarySchema.safeParse({
+        ...base,
+        mode: "ambient",
+        revision: 0,
+      }).success,
+    ).toBe(false);
+  });
+
+  test("enforces status-specific terminal timing", () => {
+    const base = {
+      id: "session-123",
+      conversationId: "conv-123",
+      mode: "browser",
+      sourceStartedAt: 100,
+      firstIncludedAt: null,
+      firstIncludedMessageId: null,
+      lastActivityAt: 125,
+      lastOwnedMessageId: null,
+      revision: 1,
+    };
+
+    expect(
+      ModeSessionSummarySchema.safeParse({
+        ...base,
+        status: "active",
+        endedAt: 150,
+        endReason: "settled",
+      }).success,
+    ).toBe(false);
+    expect(
+      ModeSessionSummarySchema.safeParse({
+        ...base,
+        status: "completed",
+        endedAt: null,
+        endReason: "settled",
+      }).success,
+    ).toBe(false);
+    expect(
+      ModeSessionSummarySchema.safeParse({
+        ...base,
+        status: "completed",
+        endedAt: 120,
+        endReason: "settled",
+      }).success,
+    ).toBe(false);
+  });
+
+  test("keeps display boundary fields paired", () => {
+    expect(
+      ModeSessionSummarySchema.safeParse({
+        id: "session-123",
+        conversationId: "conv-123",
+        mode: "ambient",
+        status: "active",
+        sourceStartedAt: 100,
+        firstIncludedAt: 110,
+        firstIncludedMessageId: null,
+        lastActivityAt: 125,
+        lastOwnedMessageId: null,
+        endedAt: null,
+        endReason: null,
+        revision: 1,
+      }).success,
+    ).toBe(false);
+  });
+});

package/src/api/mode-session.ts

@@ -0,0 +1,148 @@
+import { z } from "zod";
+
+export const ModeSessionModeSchema = z.enum([
+  "computer_use",
+  "browser",
+  "live_vision",
+  "ambient",
+]);
+export type ModeSessionMode = z.infer<typeof ModeSessionModeSchema>;
+
+/** Whether a stamped row may start the user-visible span for this mode. */
+export function modeSessionRowStartsDisplay(
+  mode: ModeSessionMode,
+  startsDisplayBoundary: boolean,
+): boolean {
+  return mode === "live_vision" || startsDisplayBoundary;
+}
+
+export const ModeSessionStatusSchema = z.enum([
+  "active",
+  "completed",
+  "interrupted",
+]);
+export type ModeSessionStatus = z.infer<typeof ModeSessionStatusSchema>;
+
+/** Immutable ownership stamped onto transcript message metadata. */
+export const ModeSessionSchema = z.object({
+  mode: ModeSessionModeSchema,
+  id: z.string().min(1),
+});
+export type ModeSession = z.infer<typeof ModeSessionSchema>;
+
+export function parseModeSession(value: unknown): ModeSession | undefined {
+  const parsed = ModeSessionSchema.safeParse(value);
+  return parsed.success ? parsed.data : undefined;
+}
+
+/** Historic or caller-authored invalid stamps behave like an absent stamp. */
+export const TolerantModeSessionSchema = z.preprocess(
+  parseModeSession,
+  ModeSessionSchema.optional(),
+);
+
+const MAX_DATE_MS = 8_640_000_000_000_000;
+
+export const ModeSessionActivitySchema = z
+  .object({
+    firstAt: z.number().int().nonnegative().max(MAX_DATE_MS),
+    lastAt: z.number().int().nonnegative().max(MAX_DATE_MS),
+  })
+  .refine((activity) => activity.lastAt >= activity.firstAt, {
+    message: "Mode session activity cannot end before it starts",
+    path: ["lastAt"],
+  });
+export type ModeSessionActivity = z.infer<typeof ModeSessionActivitySchema>;
+
+const ModeSessionSummaryBaseSchema = z.object({
+  id: z.string().min(1),
+  conversationId: z.string().min(1),
+  mode: ModeSessionModeSchema,
+  sourceStartedAt: z.number().int().nonnegative(),
+  firstIncludedAt: z.number().int().nonnegative().nullable(),
+  firstIncludedMessageId: z.string().min(1).nullable(),
+  lastActivityAt: z.number().int().nonnegative(),
+  lastOwnedMessageId: z.string().min(1).nullable(),
+  revision: z.number().int().positive(),
+});
+
+/** Durable lifecycle summary for one conversation-owned presentation run. */
+export const ModeSessionSummarySchema = z
+  .discriminatedUnion("status", [
+    ModeSessionSummaryBaseSchema.extend({
+      status: z.literal("active"),
+      endedAt: z.null(),
+      endReason: z.null(),
+    }),
+    ModeSessionSummaryBaseSchema.extend({
+      status: z.literal("completed"),
+      endedAt: z.number().int().nonnegative(),
+      endReason: z.string().min(1),
+    }),
+    ModeSessionSummaryBaseSchema.extend({
+      status: z.literal("interrupted"),
+      endedAt: z.number().int().nonnegative().nullable(),
+      endReason: z.string().min(1),
+    }),
+  ])
+  .superRefine((summary, ctx) => {
+    if (
+      (summary.firstIncludedAt === null) !==
+      (summary.firstIncludedMessageId === null)
+    ) {
+      ctx.addIssue({
+        code: "custom",
+        message: "First included time and message ID must be set together",
+      });
+    }
+    if (summary.endedAt !== null && summary.endedAt < summary.lastActivityAt) {
+      ctx.addIssue({
+        code: "custom",
+        message: "Mode session end cannot precede its last activity",
+        path: ["endedAt"],
+      });
+    }
+    if (summary.lastActivityAt < summary.sourceStartedAt) {
+      ctx.addIssue({
+        code: "custom",
+        message: "Last activity cannot precede source start",
+        path: ["lastActivityAt"],
+      });
+    }
+    if (
+      summary.firstIncludedAt !== null &&
+      summary.firstIncludedAt > summary.lastActivityAt
+    ) {
+      ctx.addIssue({
+        code: "custom",
+        message: "First included time cannot follow last activity",
+        path: ["firstIncludedAt"],
+      });
+    }
+  });
+export type ModeSessionSummary = z.infer<typeof ModeSessionSummarySchema>;
+
+export const ModeSessionRuntimeStateSchema = z.enum(["waiting", "finishing"]);
+export type ModeSessionRuntimeState = z.infer<
+  typeof ModeSessionRuntimeStateSchema
+>;
+
+/** Durable lifecycle truth plus optional process-local presentation state. */
+export const ModeSessionDescriptorSchema = z
+  .object({
+    summary: ModeSessionSummarySchema,
+    runtimeState: ModeSessionRuntimeStateSchema.optional(),
+  })
+  .superRefine((descriptor, ctx) => {
+    if (
+      descriptor.runtimeState !== undefined &&
+      descriptor.summary.status !== "active"
+    ) {
+      ctx.addIssue({
+        code: "custom",
+        message: "Only active mode sessions can expose runtime state",
+        path: ["runtimeState"],
+      });
+    }
+  });
+export type ModeSessionDescriptor = z.infer<typeof ModeSessionDescriptorSchema>;

package/src/api/remember-tool.ts

@@ -0,0 +1,15 @@
+import { z } from "zod";
+
+/**
+ * The `remember` tool's input: what the daemon parses a call with, and what a
+ * client reads a recorded call's input with. `content` is one fact or a list
+ * of them. `finish_turn` yields the turn only when it is `true`; any other
+ * value reads as absent. Unknown keys, such as the injected `activity`, pass
+ * through.
+ */
+export const RememberInputSchema = z.looseObject({
+  content: z.union([z.string(), z.array(z.string()).min(1)]),
+  finish_turn: z.boolean().optional().catch(undefined),
+});
+
+export type RememberInput = z.infer<typeof RememberInputSchema>;

package/src/api/responses/conversation-message.ts

@@ -33,6 +33,10 @@ import {
 import { AnsweredQuestionSchema } from "../events/question-answered.js";
 import { QuestionEntrySchema } from "../events/question-request.js";
 import { ToolActivityMetadataSchema } from "../events/tool-result.js";
+import {
+  ModeSessionActivitySchema,
+  ModeSessionSchema,
+} from "../mode-session.js";
 
 // ---------------------------------------------------------------------------
 // Attachment metadata
@@ -492,6 +496,8 @@ export type ConversationContentBlock = z.infer<
  */
 export const ConversationMessageSchema = z.object({
   id: z.string(),
+  modeSession: ModeSessionSchema.optional(),
+  modeSessionActivity: ModeSessionActivitySchema.optional(),
   /**
    * Server message ids folded into this display row when consecutive
    * assistant messages were consolidated for history rendering.

package/src/approvals/AGENTS.md

@@ -11,7 +11,9 @@ The end-to-end lifecycle of interactive guardian requests (approvals, questions)
 
 ## Single-Guardian Invariant
 
-Each assistant instance serves exactly one guardian. Multi-guardian is not supported and will never be. All connections, browser sessions, approval channels, and trust contexts within a single assistant process belong to the same guardian principal. Do not introduce guardian-keyed maps, per-guardian routing logic, or multi-guardian multiplexing — they add complexity without a real use case and create the false impression that cross-guardian isolation is required.
+Each assistant instance serves exactly one guardian. Multi-guardian is not supported and will never be. Only that guardian's principal decides guardian requests: `applyGuardianDecision()` requires it exactly. Do not introduce guardian-keyed maps, per-guardian routing logic, or multi-guardian multiplexing; they add complexity without a real use case and create the false impression that cross-guardian isolation is required.
+
+One guardian does not mean one person connected. The event stream (`GET /v1/events`) admits any actor principal with `chat.read`, so a non-guardian actor can hold a valid connection. Routes that must be guardian-only set `requireGuardian: true`, which `routes/http-adapter.ts` enforces with `requireBoundGuardian` (`runtime/auth/require-bound-guardian.ts`). Content meant only for the guardian is scoped to the guardian's principal at delivery (`targetActorPrincipalId` on the event hub), never assumed safe because it reached a connection.
 
 ## Guardian Verification Invariant
 
@@ -21,7 +23,7 @@ Conversational guardian verification control-plane invocation is guardian-only.
 
 ## Memory Provenance Invariant
 
-All memory retrieval decisions must consider actor-role provenance. Untrusted actors (non-guardian, unverified_channel) must not receive memory recall results. This invariant is enforced in `indexer.ts` (write gate) and the memory plugin's `injectors.ts` (read gate), which admits personal-memory content only for a turn whose trust context passes `isPersonalMemoryAllowed`.
+Memory access follows the acting actor's capabilities: `resolveCapabilities(trustClass).canAccessMemory`, which only `guardian` holds. An actor without it neither reads nor writes the guardian's memory. The memory plugin's `injectors.ts` admits personal-memory content only for a turn whose trust context passes `isPersonalMemoryAllowed`; the `recall`, `remember`, and `delete_memory_page` tools refuse such a turn (`plugins/defaults/memory/tools.ts`); and extraction from conversation messages runs only for guardian-provenance rows, or legacy rows with none (`plugins/defaults/memory/indexer.ts`).
 
 ## Guardian Privilege Isolation Invariant
 

package/src/browser/__tests__/operations.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 
 import {
   BROWSER_OPERATION_META,
+  browserOperationLifecycle,
   executeBrowserOperation,
 } from "../operations.js";
 import { BROWSER_OPERATIONS, type BrowserOperation } from "../types.js";
@@ -19,6 +20,34 @@ describe("browser operations contract", () => {
     expect(BROWSER_OPERATION_META).toHaveLength(BROWSER_OPERATIONS.length);
   });
 
+  test("classifies every typed operation for session lifecycle", () => {
+    const classifications = Object.fromEntries(
+      BROWSER_OPERATIONS.map((operation) => [
+        operation,
+        browserOperationLifecycle(operation),
+      ]),
+    );
+    expect(classifications).toEqual({
+      navigate: "action",
+      snapshot: "action",
+      screenshot: "action",
+      close: "terminal",
+      attach: "action",
+      detach: "terminal",
+      click: "action",
+      type: "action",
+      press_key: "action",
+      scroll: "action",
+      select_option: "action",
+      hover: "action",
+      wait_for: "action",
+      extract: "action",
+      wait_for_download: "action",
+      fill_credential: "action",
+      status: "status",
+    });
+  });
+
   // ── Every operation has a dispatch handler ─────────────────────────
 
   test("every operation has a dispatch handler (rejects unknown)", async () => {

package/src/browser/operation-meta.ts

@@ -216,7 +216,7 @@ Examples:
         name: "key",
         type: "string",
         description:
-          'The key to press (e.g. "Enter", "Escape", "Tab", "ArrowDown").',
+          'The key or modifier chord to press (e.g. "Enter", "Meta+v", "Control+a").',
         required: true,
       },
       {
@@ -238,6 +238,7 @@ the key is dispatched to that element; otherwise to the focused element.
 Examples:
   $ assistant browser press-key --key Enter
   $ assistant browser press-key --key Tab --element-id e5
+  $ assistant browser press-key --key "Meta+v"
   $ assistant browser press-key --key Escape`,
   },
   {

package/src/browser/operations.ts

@@ -36,6 +36,35 @@ import type { ToolExecutionResult } from "../tools/types.js";
 import type { BrowserOperationContext as ToolContext } from "./types.js";
 import type { BrowserOperation } from "./types.js";
 
+export type BrowserOperationLifecycle = "action" | "terminal" | "status";
+
+const OPERATION_LIFECYCLE: Record<BrowserOperation, BrowserOperationLifecycle> =
+  {
+    navigate: "action",
+    snapshot: "action",
+    screenshot: "action",
+    close: "terminal",
+    attach: "action",
+    detach: "terminal",
+    click: "action",
+    type: "action",
+    press_key: "action",
+    scroll: "action",
+    select_option: "action",
+    hover: "action",
+    wait_for: "action",
+    extract: "action",
+    wait_for_download: "action",
+    fill_credential: "action",
+    status: "status",
+  };
+
+export function browserOperationLifecycle(
+  operation: BrowserOperation,
+): BrowserOperationLifecycle {
+  return OPERATION_LIFECYCLE[operation];
+}
+
 // ── Dispatch handlers ────────────────────────────────────────────────
 
 /**

package/src/browser/virtual-desktop-target.ts

@@ -1,4 +1,4 @@
-import { isVirtualDesktopEnabled } from "../desktop/virtual-desktop-feature.js";
+import { shouldUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
 import { browserManager } from "../tools/browser/browser-manager.js";
 import { normalizeBrowserMode } from "../tools/browser/browser-mode.js";
 import { getPinnedTab } from "../tools/browser/pinned-tabs.js";
@@ -18,16 +18,10 @@ export function shouldUseVirtualDesktopBrowser(
     mode.mode !== "auto" ||
     input.target_client_id ||
     input.use_active_tab ||
-    context.transportInterface !== "web" ||
-    context.clientOs === "macos" ||
-    context.clientOs === "windows" ||
-    context.clientOs === "linux" ||
-    context.trustClass !== "guardian" ||
-    !context.sourceActorPrincipalId ||
     browserManager.getPreferredBackendKind(context.conversationId) !== null ||
     getPinnedTab(context.conversationId)
   ) {
     return false;
   }
-  return isVirtualDesktopEnabled();
+  return shouldUseVirtualDesktop(context);
 }

package/src/bundler/bundle-signer.ts

@@ -115,7 +115,7 @@ export async function signBundle(
   const { signature, keyId } = await requestSignature(canonicalPayload);
 
   // 5. Build SignatureJson
-  const signatureJson: SignatureJson = {
+  return {
     algorithm: "ed25519",
     signer: {
       key_id: keyId,
@@ -124,6 +124,4 @@ export async function signBundle(
     content_hashes: contentHashes,
     signature,
   };
-
-  return signatureJson;
 }

package/src/calls/__tests__/barge-in-guard.test.ts

@@ -12,13 +12,16 @@ describe("createBargeInGuard", () => {
     expect(guard.track("speech", 100)).toBe("pending");
     expect(guard.track("speech", 100)).toBe("pending");
     expect(guard.speechMs).toBe(200);
+    expect(guard.fired).toBe(false);
   });
 
   test("sustained speech reaching the threshold fires once and stays fired", () => {
     const guard = createBargeInGuard(250);
     guard.track("speech", 200);
     expect(guard.track("speech", 50)).toBe("fired");
+    expect(guard.fired).toBe(true);
     expect(guard.track("silence", 1_000)).toBe("fired");
+    expect(guard.fired).toBe(true);
   });
 
   test("a brief sub-threshold gap does not reset the run", () => {

package/src/calls/__tests__/call-controller.test.ts

@@ -421,6 +421,7 @@ function setupController(
 ) {
   ensureConversation("conv-ctrl-test");
   const session = createCallSession({
+    direction: "inbound",
     conversationId: "conv-ctrl-test",
     provider: "twilio",
     fromNumber: "+15551111111",
@@ -471,6 +472,7 @@ function setupControllerWithOrigin(task?: string) {
   ensureConversation("conv-ctrl-voice");
   ensureConversation("conv-ctrl-origin");
   const session = createCallSession({
+    direction: "inbound",
     conversationId: "conv-ctrl-voice",
     provider: "twilio",
     fromNumber: "+15551111111",
@@ -5028,4 +5030,96 @@ describe("call-controller", () => {
       await turnPromise.catch(() => {});
     });
   });
+  describe("turn metrics", () => {
+    test("a caller turn records the marks the latency numbers are built from", async () => {
+      const { controller } = setupController();
+
+      await controller.handleCallerUtterance("What is the weather?");
+
+      const snapshot = controller.getMetricsSnapshot();
+      expect(snapshot.recentTurns.length).toBe(1);
+      const turn = snapshot.recentTurns[0];
+      expect(turn.status).toBe("completed");
+      // The transcript mark is seeded from when the caller stopped talking,
+      // the dispatch mark from when the leg went out, and the delta mark from
+      // the first token back: the three anchors every phone latency number is
+      // subtracted from.
+      expect(turn.timestamps.finalTranscriptAtMs).not.toBeNull();
+      expect(turn.timestamps.assistantDispatchAtMs).not.toBeNull();
+      expect(turn.timestamps.firstAssistantDeltaAtMs).not.toBeNull();
+      expect(turn.durations.dispatchToFirstAssistantDeltaMs).not.toBeNull();
+      expect(turn.durations.totalTurnDurationMs).not.toBeNull();
+      // The provider's final doubles as the speech-end anchor, so the turn's
+      // round trip is measurable on phone as it is in live voice.
+      expect(turn.timestamps.utteranceEndAtMs).not.toBeNull();
+      expect(turn.durations.roundTripMs).not.toBeNull();
+
+      controller.destroy();
+    });
+
+    test("the transcript mark predates the dispatch it triggered", async () => {
+      // Seeding backdates the turn to the caller's own boundary; a turn that
+      // opened at dispatch time would report a felt latency that starts after
+      // the caller already finished speaking.
+      const { controller } = setupController();
+
+      await controller.handleCallerUtterance("Hello there");
+
+      const turn = controller.getMetricsSnapshot().recentTurns[0];
+      const finalTranscriptAtMs = turn.timestamps.finalTranscriptAtMs;
+      const assistantDispatchAtMs = turn.timestamps.assistantDispatchAtMs;
+      expect(finalTranscriptAtMs).not.toBeNull();
+      expect(assistantDispatchAtMs).not.toBeNull();
+      expect(finalTranscriptAtMs!).toBeLessThanOrEqual(assistantDispatchAtMs!);
+      expect(turn.timestamps.startedAtMs).toBeLessThanOrEqual(
+        finalTranscriptAtMs!,
+      );
+
+      controller.destroy();
+    });
+
+    test("a barge-in ends the turn once, as cancelled", async () => {
+      // One interruption is one cancelled turn: the count downstream reads as
+      // "turns the caller cut off", so a second settle would double it.
+      // A turn that never emits or completes holds the controller in the
+      // pre-speech phase, which a sustained barge-in can still cut off.
+      mockStartVoiceTurn.mockImplementation(
+        async (opts: { onComplete: () => void }) => ({
+          turnId: "run-metrics-slow",
+          abort: () => opts.onComplete(),
+        }),
+      );
+      const { controller } = setupController();
+      const turnPromise = controller.handleCallerUtterance("Tell me a story");
+      for (let i = 0; i < 5; i++) {
+        await Promise.resolve();
+      }
+      expect(controller.getState()).toBe("processing");
+
+      expect(controller.handleBargeIn()).toBe(true);
+      await turnPromise.catch(() => {});
+      controller.destroy();
+
+      const turns = controller.getMetricsSnapshot().recentTurns;
+      expect(turns.length).toBe(1);
+      expect(turns[0].status).toBe("cancelled");
+      expect(turns[0].cancellationReason).toBe("interrupted");
+      expect(turns[0].timestamps.bargeInAtMs).not.toBeNull();
+    });
+
+    test("no turn is invented between turns", async () => {
+      // Every mark opens a turn when none is active, so a barge-in on an idle
+      // controller must leave the collector empty rather than start a turn
+      // that never settles.
+      const { controller } = setupController();
+
+      expect(controller.handleBargeIn()).toBe(false);
+
+      const snapshot = controller.getMetricsSnapshot();
+      expect(snapshot.recentTurns.length).toBe(0);
+      expect(snapshot.activeTurn).toBeNull();
+
+      controller.destroy();
+    });
+  });
 });