@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/ipc/__tests__/browser-ipc.test.ts

@@ -16,7 +16,11 @@ import {
   test,
 } from "bun:test";
 
-import { desktopDependencyInstaller } from "../../desktop/desktop-dependencies.js";
+import { setOverridesForTesting } from "../../__tests__/feature-flag-test-helpers.js";
+import { isSessionGroupsEnabled } from "../../config/session-groups-gate.js";
+import type { BrowserOperationToken } from "../../daemon/browser-mode-session.js";
+import { BrowserModeSessionProducer } from "../../daemon/browser-mode-session.js";
+import { desktopDependencies } from "../../desktop/desktop-dependencies.js";
 import * as desktopFeature from "../../desktop/virtual-desktop-feature.js";
 import { browserManager } from "../../tools/browser/browser-manager.js";
 import type { ToolExecutionResult } from "../../tools/types.js";
@@ -46,11 +50,69 @@ let mockConversation: {
   clientOs?: string;
   getTurnActorPrincipalId?: () => string | undefined;
   abortController?: AbortController;
+  currentRequestId?: string;
+  browserModeSessions?: {
+    beginOperation: (input: {
+      turnId: string;
+      lifecycle: "action" | "terminal" | "status";
+      at: number;
+    }) => BrowserOperationToken | undefined;
+    finishOperation: (
+      token: BrowserOperationToken | undefined,
+      outcome: {
+        at: number;
+        isError: boolean;
+        cancelled: boolean;
+        terminalReason?: "browser_closed" | "browser_detached";
+      },
+    ) => boolean;
+  };
 } | null = null;
 
+const lifecycleBegins: Array<{
+  turnId: string;
+  lifecycle: "action" | "terminal" | "status";
+  at: number;
+}> = [];
+const lifecycleFinishes: Array<{
+  token: BrowserOperationToken | undefined;
+  outcome: {
+    at: number;
+    isError: boolean;
+    cancelled: boolean;
+    terminalReason?: "browser_closed" | "browser_detached";
+  };
+}> = [];
+
+function browserLifecycle() {
+  return {
+    beginOperation(input: (typeof lifecycleBegins)[number]) {
+      lifecycleBegins.push(input);
+      return {
+        turnId: input.turnId,
+        lifecycle: input.lifecycle === "status" ? "action" : input.lifecycle,
+        owner: { id: "browser-session", mode: "browser" as const },
+      } as BrowserOperationToken;
+    },
+    finishOperation(
+      token: BrowserOperationToken | undefined,
+      outcome: (typeof lifecycleFinishes)[number]["outcome"],
+    ) {
+      lifecycleFinishes.push({ token, outcome });
+      return true;
+    },
+  };
+}
+
 let mockFindConversationCalls: string[] = [];
 
 mock.module("../../browser/operations.js", () => ({
+  browserOperationLifecycle: (operation: string) =>
+    operation === "status"
+      ? "status"
+      : operation === "close" || operation === "detach"
+        ? "terminal"
+        : "action",
   executeBrowserOperation: async (
     operation: string,
     input: Record<string, unknown>,
@@ -78,6 +140,7 @@ mock.module("../../daemon/conversation-registry.js", () => ({
     mockFindConversationCalls.push(conversationId);
     return mockConversation ?? undefined;
   },
+  findConversationOrSubagent: () => mockConversation ?? undefined,
 }));
 
 let desktopEnabled = false;
@@ -87,10 +150,9 @@ const enabledSpy = spyOn(
   desktopFeature,
   "isVirtualDesktopEnabled",
 ).mockImplementation(() => desktopEnabled);
-const readySpy = spyOn(
-  desktopDependencyInstaller,
-  "getStatus",
-).mockImplementation(() => ({ state: desktopReady ? "ready" : "required" }));
+const readySpy = spyOn(desktopDependencies, "getStatus").mockImplementation(
+  () => ({ state: desktopReady ? "ready" : "failed" }),
+);
 afterAll(() => {
   enabledSpy.mockRestore();
   readySpy.mockRestore();
@@ -147,6 +209,7 @@ function callHandler(
 // ---------------------------------------------------------------------------
 
 afterEach(() => {
+  setOverridesForTesting({});
   desktopEnabled = false;
   desktopReady = false;
   desktopFailure = false;
@@ -156,6 +219,8 @@ afterEach(() => {
   mockOperationCalls = [];
   mockConversation = null;
   mockFindConversationCalls = [];
+  lifecycleBegins.length = 0;
+  lifecycleFinishes.length = 0;
 });
 
 // ---------------------------------------------------------------------------
@@ -471,6 +536,150 @@ describe("browser_execute route", () => {
     });
   });
 
+  test("brackets a typed live-turn action with browser lifecycle", async () => {
+    mockConversation = {
+      currentRequestId: "turn-123",
+      browserModeSessions: browserLifecycle(),
+      getTurnActorPrincipalId: () => undefined,
+    };
+
+    await callHandler({
+      operation: "navigate",
+      input: { url: "https://example.com" },
+      conversationId: "conv-live",
+    });
+
+    expect(lifecycleBegins).toEqual([
+      expect.objectContaining({ turnId: "turn-123", lifecycle: "action" }),
+    ]);
+    expect(lifecycleFinishes).toEqual([
+      expect.objectContaining({
+        token: expect.objectContaining({ turnId: "turn-123" }),
+        outcome: expect.objectContaining({
+          isError: false,
+          cancelled: false,
+        }),
+      }),
+    ]);
+  });
+
+  test("executes a live-turn action without tracking while session groups are disabled", async () => {
+    setOverridesForTesting({ "session-groups": false });
+    const activateSource = mock(() => {
+      throw new Error("disabled tracking must not activate a source");
+    });
+    mockConversation = {
+      currentRequestId: "turn-123",
+      browserModeSessions: new BrowserModeSessionProducer(
+        {
+          activateSource,
+          claimTurn: mock(() => undefined),
+          getTurnOwner: mock(() => undefined),
+          recordActivity: mock(() => false),
+          retireSource: mock(() => false),
+        },
+        1,
+        isSessionGroupsEnabled,
+      ),
+      getTurnActorPrincipalId: () => undefined,
+    };
+
+    const result = await callHandler({
+      operation: "navigate",
+      input: { url: "https://example.com" },
+      conversationId: "conv-live",
+    });
+
+    expect(result).toEqual({ content: "ok", isError: false });
+    expect(mockOperationCalls).toHaveLength(1);
+    expect(activateSource).not.toHaveBeenCalled();
+  });
+
+  test.each(["navigate", "close"] as const)(
+    "executes %s once after tracking admission throws",
+    async (operation) => {
+      mockConversation = {
+        currentRequestId: "turn-123",
+        browserModeSessions: {
+          ...browserLifecycle(),
+          beginOperation() {
+            throw new Error("tracking unavailable");
+          },
+        },
+        getTurnActorPrincipalId: () => undefined,
+      };
+      const result = await callHandler({
+        operation,
+        conversationId: "conv-live",
+      });
+      expect(result).toEqual({ content: "ok", isError: false });
+      expect(mockOperationCalls).toHaveLength(1);
+    },
+  );
+
+  test("passes typed terminal success and failure outcomes", async () => {
+    mockConversation = {
+      currentRequestId: "turn-123",
+      browserModeSessions: browserLifecycle(),
+      getTurnActorPrincipalId: () => undefined,
+    };
+
+    await callHandler({ operation: "detach", conversationId: "conv-live" });
+    expect(lifecycleBegins[0]).toMatchObject({ lifecycle: "terminal" });
+    expect(lifecycleFinishes[0]?.outcome).toMatchObject({
+      isError: false,
+      terminalReason: "browser_detached",
+    });
+
+    mockOperationResult = { content: "close failed", isError: true };
+    await callHandler({ operation: "close", conversationId: "conv-live" });
+    expect(lifecycleFinishes[1]?.outcome).toMatchObject({
+      isError: true,
+      terminalReason: "browser_closed",
+    });
+  });
+
+  test.each(["navigate", "close"] as const)(
+    "preserves a successful %s result when session tracking fails",
+    async (operation) => {
+      mockOperationResult = {
+        content: `${operation} completed`,
+        isError: false,
+      };
+      const lifecycle = browserLifecycle();
+      mockConversation = {
+        currentRequestId: "turn-123",
+        browserModeSessions: {
+          ...lifecycle,
+          finishOperation(token, outcome) {
+            lifecycle.finishOperation(token, outcome);
+            throw new Error("session tracking unavailable");
+          },
+        },
+        getTurnActorPrincipalId: () => undefined,
+      };
+
+      const result = await callHandler({
+        operation,
+        input: operation === "navigate" ? { url: "https://example.com" } : {},
+        conversationId: "conv-live",
+      });
+
+      expect(result).toEqual({
+        content: `${operation} completed`,
+        isError: false,
+      });
+      expect(mockOperationCalls).toHaveLength(1);
+      expect(lifecycleFinishes).toHaveLength(1);
+    },
+  );
+
+  test("does not observe lifecycle for a standalone CLI operation", async () => {
+    await callHandler({ operation: "navigate", sessionId: "standalone" });
+    expect(lifecycleBegins).toEqual([]);
+    expect(lifecycleFinishes).toEqual([]);
+  });
+
   // ── Input defaults ─────────────────────────────────────────────────
 
   test("defaults input to empty object when omitted", async () => {
@@ -639,7 +848,7 @@ test("a streamed browser failure does not switch to Playwright or personal Chrom
   expect(mockOperationCalls).toHaveLength(0);
 });
 
-test("first web browser use routes to virtual desktop setup before installation", async () => {
+test("missing image components do not redirect web browser use to a personal browser", async () => {
   webConversation();
   desktopReady = false;
   await callHandler({

package/src/ipc/assistant-server.ts

@@ -59,6 +59,7 @@ import { RouteResponse } from "../runtime/routes/types.js";
 import { getLogger } from "../util/logger.js";
 import { mapGatewayIpcConnectError } from "./gateway-ipc-errors.js";
 import { ACTIVATION_SYNC_IPC_METHODS } from "./routes/activation-sync-ipc-routes.js";
+import { CHANNEL_REPLY_IPC_METHODS } from "./routes/channel-reply-ipc-routes.js";
 import { CONTACTS_INFO_IPC_METHODS } from "./routes/contacts-info-ipc-routes.js";
 import { CONTACTS_MIRROR_IPC_METHODS } from "./routes/contacts-mirror-ipc-routes.js";
 import { CONVERSATION_SYNC_IPC_METHODS } from "./routes/conversation-sync-ipc-routes.js";
@@ -216,6 +217,7 @@ export class AssistantIpcServer {
     // never in ROUTES.
     for (const methodMap of [
       INVITE_IPC_METHODS,
+      CHANNEL_REPLY_IPC_METHODS,
       CONTACTS_INFO_IPC_METHODS,
       CONTACTS_MIRROR_IPC_METHODS,
       GUARDIAN_LABEL_IPC_METHODS,

package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts

@@ -0,0 +1,111 @@
+/**
+ * The gateway's replies to messages it answered itself reach the person
+ * through the channel transport, the same one a turn's reply uses.
+ *
+ * The Bot API call is the boundary: everything between the gateway's request
+ * and Telegram's `sendMessage` is the real code, so the assertion is that the
+ * text a person would read left for their chat, in the topic they wrote from.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { DELIVER_GATEWAY_REPLY_IPC_METHOD } from "@vellumai/gateway-client";
+
+type CallTelegramBotApi =
+  typeof import("../../../messaging/providers/telegram-bot/api.js").callTelegramBotApi;
+
+const callTelegramBotApiMock = mock<CallTelegramBotApi>(
+  async () => ({ message_id: 42 }) as never,
+);
+
+// Spread the actual module so exports this suite does not touch stay
+// importable by files that share the bun process.
+const actualTelegramApi =
+  await import("../../../messaging/providers/telegram-bot/api.js");
+mock.module("../../../messaging/providers/telegram-bot/api.js", () => ({
+  ...actualTelegramApi,
+  callTelegramBotApi: (method: string, body: Record<string, unknown>) =>
+    callTelegramBotApiMock(method, body),
+}));
+
+const { CHANNEL_REPLY_IPC_METHODS } =
+  await import("../channel-reply-ipc-routes.js");
+const { BadGatewayError, BadRequestError } =
+  await import("../../../runtime/routes/errors.js");
+
+const deliverGatewayReply =
+  CHANNEL_REPLY_IPC_METHODS[DELIVER_GATEWAY_REPLY_IPC_METHOD]!;
+
+// The callback URL exactly as the gateway's Telegram webhook builds it for a
+// message sent in a topic.
+const TELEGRAM_CALLBACK = "http://127.0.0.1:7830/deliver/telegram?threadId=7";
+
+beforeEach(() => {
+  callTelegramBotApiMock.mockClear();
+});
+
+describe("deliver_gateway_reply", () => {
+  test("sends the reply to the person's Telegram chat, in their topic", async () => {
+    const result = await deliverGatewayReply({
+      body: {
+        callbackUrl: TELEGRAM_CALLBACK,
+        chatId: "12345",
+        text: "Welcome! You've been granted access.",
+        assistantId: "self",
+      },
+    });
+
+    const sends = callTelegramBotApiMock.mock.calls.filter(
+      ([method]) => method === "sendMessage",
+    );
+    expect(sends).toHaveLength(1);
+    expect(sends[0]![1]).toMatchObject({
+      chat_id: "12345",
+      text: "Welcome! You've been granted access.",
+      message_thread_id: 7,
+    });
+    expect(result).toEqual({ ok: true, messageIds: ["42"] });
+  });
+
+  test("answers a send the channel refuses with a status, not a bare error", async () => {
+    // Only a RouteError crosses IPC with a status; without one the gateway
+    // reads the refusal as a daemon that never answered.
+    callTelegramBotApiMock.mockImplementationOnce(async () => {
+      throw new Error("Bad Request: chat not found");
+    });
+
+    const rejection = expect(
+      deliverGatewayReply({
+        body: {
+          callbackUrl: TELEGRAM_CALLBACK,
+          chatId: "12345",
+          text: "Welcome! You've been granted access.",
+        },
+      }),
+    ).rejects;
+    await rejection.toBeInstanceOf(BadGatewayError);
+    await rejection.toMatchObject({ statusCode: 502 });
+  });
+
+  test("refuses a callback no channel transport owns, sending nothing", async () => {
+    await expect(
+      deliverGatewayReply({
+        body: {
+          callbackUrl: "http://127.0.0.1:7830/deliver/email",
+          chatId: "someone@example.com",
+          text: "Verification successful.",
+        },
+      }),
+    ).rejects.toBeInstanceOf(BadRequestError);
+    expect(callTelegramBotApiMock).not.toHaveBeenCalled();
+  });
+
+  test("refuses a request with no text, sending nothing", async () => {
+    await expect(
+      deliverGatewayReply({
+        body: { callbackUrl: TELEGRAM_CALLBACK, chatId: "12345", text: "" },
+      }),
+    ).rejects.toBeInstanceOf(BadRequestError);
+    expect(callTelegramBotApiMock).not.toHaveBeenCalled();
+  });
+});

package/src/ipc/routes/channel-reply-ipc-routes.ts

@@ -0,0 +1,73 @@
+/**
+ * IPC-only channel reply method called by the gateway over the assistant IPC
+ * socket (`ipcCallAssistant`).
+ *
+ * The gateway answers some inbound messages itself without forwarding them
+ * (a verification code, an invite redemption), so the assistant never sees
+ * them. Its reply still has to reach the person, and the channel transports
+ * that can send it live here. This hands the reply to the transport the
+ * inbound message's callback URL names, which is how the daemon's own
+ * intercepts answer an inbound message: the gateway sends through the same
+ * transports and never to a provider itself.
+ *
+ * Nothing is recorded in a conversation. The message the reply answers never
+ * entered one, so a row for the reply would stand alone in a transcript that
+ * does not hold what it responds to.
+ */
+
+import {
+  DELIVER_GATEWAY_REPLY_IPC_METHOD,
+  GatewayReplyRequestSchema,
+} from "@vellumai/gateway-client";
+
+import {
+  deliverDirect,
+  isDirectDelivery,
+} from "../../messaging/providers/index.js";
+import {
+  BadGatewayError,
+  BadRequestError,
+} from "../../runtime/routes/errors.js";
+import { parseBody } from "../../runtime/routes/parse-body.js";
+import type { RouteHandlerArgs } from "../../runtime/routes/types.js";
+
+/**
+ * Deliver a gateway-composed reply through the channel transport its callback
+ * URL names. Refuses a callback no transport owns rather than fetching it, so
+ * the gateway learns the channel cannot be answered this way.
+ *
+ * A send the channel refuses is answered as a `BadGatewayError`, as
+ * `channels/send` answers it: only a `RouteError` carries a status over IPC,
+ * and the gateway reads an error without one as a daemon that never answered.
+ */
+export async function handleDeliverGatewayReply({
+  body = {},
+}: RouteHandlerArgs) {
+  const { callbackUrl, chatId, text, assistantId } = parseBody(
+    GatewayReplyRequestSchema,
+    body,
+  );
+  if (!isDirectDelivery(callbackUrl)) {
+    throw new BadRequestError("No channel transport owns this callback URL");
+  }
+  try {
+    return await deliverDirect(callbackUrl, {
+      chatId,
+      text,
+      ...(assistantId ? { assistantId } : {}),
+    });
+  } catch (err) {
+    throw new BadGatewayError(err instanceof Error ? err.message : String(err));
+  }
+}
+
+/**
+ * IPC-only channel reply methods, keyed by IPC operationId. Registered
+ * directly on the assistant IPC server (see `assistant-server.ts`).
+ */
+export const CHANNEL_REPLY_IPC_METHODS: Record<
+  string,
+  (args: RouteHandlerArgs) => unknown
+> = {
+  [DELIVER_GATEWAY_REPLY_IPC_METHOD]: handleDeliverGatewayReply,
+};

package/src/live-voice/__tests__/continuation-judge.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, test } from "bun:test";
+
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+} from "../../providers/types.js";
+import {
+  CONTINUATION_JUDGE_KEEP_THRESHOLD,
+  createContinuationJudge,
+  VOICE_CONTINUATION_JUDGE_CALL_SITE,
+} from "../continuation-judge.js";
+
+function jevResponse(noul: number): ProviderResponse {
+  const answers = { answer: { type: "noul", noul } };
+  return {
+    content: [{ type: "text", text: JSON.stringify(answers) }],
+    model: "jev-latest",
+    usage: { inputTokens: 1, outputTokens: 1 },
+    stopReason: "end_turn",
+    rawResponse: { model: "jev-latest", answers },
+  };
+}
+
+function fakeProvider(
+  respond: (
+    messages: Message[],
+    options?: SendMessageOptions,
+  ) => Promise<ProviderResponse>,
+): Provider {
+  return { name: "typesafe", sendMessage: respond } as Provider;
+}
+
+const baseArgs = {
+  parentConversationId: "conv-1",
+  interruptedRequest: "Can you hit join now for me?",
+  signal: new AbortController().signal,
+};
+
+describe("createContinuationJudge", () => {
+  test("drops below the threshold and asks with both utterances", async () => {
+    let seen: { body: unknown; options?: SendMessageOptions } | null = null;
+    const judge = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async (messages, options) => {
+          const block = messages[0]!.content[0]!;
+          seen = {
+            body: JSON.parse(block.type === "text" ? block.text : ""),
+            options,
+          };
+          return jevResponse(0.09);
+        }),
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      interruption: Promise.resolve("Actually just point it out for me."),
+    });
+
+    expect(judgement).toMatchObject({
+      keep: false,
+      outcome: "drop",
+      noul: 0.09,
+    });
+    const request = seen as unknown as {
+      body: { state: Record<string, string> };
+      options: SendMessageOptions;
+    };
+    expect(request.body.state).toEqual({
+      interrupted_request: "Can you hit join now for me?",
+      caller_then_said: "Actually just point it out for me.",
+    });
+    expect(
+      (request.options.config as { callSite?: string } | undefined)?.callSite,
+    ).toBe(VOICE_CONTINUATION_JUDGE_CALL_SITE);
+  });
+
+  test("keeps at or above the threshold", async () => {
+    const judge = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async () =>
+          jevResponse(CONTINUATION_JUDGE_KEEP_THRESHOLD),
+        ),
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      interruption: Promise.resolve("Oh, also, what's the weather?"),
+    });
+
+    expect(judgement).toMatchObject({ keep: true, outcome: "keep" });
+  });
+
+  test("without a TypeSafe provider it keeps and never waits on the interruption", async () => {
+    const judge = createContinuationJudge({
+      resolveProvider: async () => null,
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      // Never settles: the judge must not await it.
+      interruption: new Promise<string | null>(() => {}),
+    });
+
+    expect(judgement).toMatchObject({ keep: true, outcome: "unavailable" });
+  });
+
+  test("keeps when no interruption arrived in time", async () => {
+    let asked = false;
+    const judge = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async () => {
+          asked = true;
+          return jevResponse(0.05);
+        }),
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      interruption: Promise.resolve(null),
+    });
+
+    expect(judgement).toMatchObject({
+      keep: true,
+      outcome: "no_interruption",
+    });
+    expect(asked).toBe(false);
+  });
+
+  test("a provider error or timeout keeps the continuation", async () => {
+    const failing = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async () => {
+          throw new Error("503");
+        }),
+    });
+    const hanging = createContinuationJudge({
+      timeoutMs: 20,
+      resolveProvider: async () =>
+        fakeProvider(async () => new Promise<ProviderResponse>(() => {})),
+    });
+
+    expect(
+      await failing({ ...baseArgs, interruption: Promise.resolve("stop") }),
+    ).toMatchObject({ keep: true, outcome: "error" });
+    expect(
+      await hanging({ ...baseArgs, interruption: Promise.resolve("stop") }),
+    ).toMatchObject({ keep: true, outcome: "timeout" });
+  });
+});