@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts

@@ -0,0 +1,111 @@
+/**
+ * The gateway's replies to messages it answered itself reach the person
+ * through the channel transport, the same one a turn's reply uses.
+ *
+ * The Bot API call is the boundary: everything between the gateway's request
+ * and Telegram's `sendMessage` is the real code, so the assertion is that the
+ * text a person would read left for their chat, in the topic they wrote from.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { DELIVER_GATEWAY_REPLY_IPC_METHOD } from "@vellumai/gateway-client";
+
+type CallTelegramBotApi =
+  typeof import("../../../messaging/providers/telegram-bot/api.js").callTelegramBotApi;
+
+const callTelegramBotApiMock = mock<CallTelegramBotApi>(
+  async () => ({ message_id: 42 }) as never,
+);
+
+// Spread the actual module so exports this suite does not touch stay
+// importable by files that share the bun process.
+const actualTelegramApi =
+  await import("../../../messaging/providers/telegram-bot/api.js");
+mock.module("../../../messaging/providers/telegram-bot/api.js", () => ({
+  ...actualTelegramApi,
+  callTelegramBotApi: (method: string, body: Record<string, unknown>) =>
+    callTelegramBotApiMock(method, body),
+}));
+
+const { CHANNEL_REPLY_IPC_METHODS } =
+  await import("../channel-reply-ipc-routes.js");
+const { BadGatewayError, BadRequestError } =
+  await import("../../../runtime/routes/errors.js");
+
+const deliverGatewayReply =
+  CHANNEL_REPLY_IPC_METHODS[DELIVER_GATEWAY_REPLY_IPC_METHOD]!;
+
+// The callback URL exactly as the gateway's Telegram webhook builds it for a
+// message sent in a topic.
+const TELEGRAM_CALLBACK = "http://127.0.0.1:7830/deliver/telegram?threadId=7";
+
+beforeEach(() => {
+  callTelegramBotApiMock.mockClear();
+});
+
+describe("deliver_gateway_reply", () => {
+  test("sends the reply to the person's Telegram chat, in their topic", async () => {
+    const result = await deliverGatewayReply({
+      body: {
+        callbackUrl: TELEGRAM_CALLBACK,
+        chatId: "12345",
+        text: "Welcome! You've been granted access.",
+        assistantId: "self",
+      },
+    });
+
+    const sends = callTelegramBotApiMock.mock.calls.filter(
+      ([method]) => method === "sendMessage",
+    );
+    expect(sends).toHaveLength(1);
+    expect(sends[0]![1]).toMatchObject({
+      chat_id: "12345",
+      text: "Welcome! You've been granted access.",
+      message_thread_id: 7,
+    });
+    expect(result).toEqual({ ok: true, messageIds: ["42"] });
+  });
+
+  test("answers a send the channel refuses with a status, not a bare error", async () => {
+    // Only a RouteError crosses IPC with a status; without one the gateway
+    // reads the refusal as a daemon that never answered.
+    callTelegramBotApiMock.mockImplementationOnce(async () => {
+      throw new Error("Bad Request: chat not found");
+    });
+
+    const rejection = expect(
+      deliverGatewayReply({
+        body: {
+          callbackUrl: TELEGRAM_CALLBACK,
+          chatId: "12345",
+          text: "Welcome! You've been granted access.",
+        },
+      }),
+    ).rejects;
+    await rejection.toBeInstanceOf(BadGatewayError);
+    await rejection.toMatchObject({ statusCode: 502 });
+  });
+
+  test("refuses a callback no channel transport owns, sending nothing", async () => {
+    await expect(
+      deliverGatewayReply({
+        body: {
+          callbackUrl: "http://127.0.0.1:7830/deliver/email",
+          chatId: "someone@example.com",
+          text: "Verification successful.",
+        },
+      }),
+    ).rejects.toBeInstanceOf(BadRequestError);
+    expect(callTelegramBotApiMock).not.toHaveBeenCalled();
+  });
+
+  test("refuses a request with no text, sending nothing", async () => {
+    await expect(
+      deliverGatewayReply({
+        body: { callbackUrl: TELEGRAM_CALLBACK, chatId: "12345", text: "" },
+      }),
+    ).rejects.toBeInstanceOf(BadRequestError);
+    expect(callTelegramBotApiMock).not.toHaveBeenCalled();
+  });
+});

package/src/ipc/routes/channel-reply-ipc-routes.ts

@@ -0,0 +1,73 @@
+/**
+ * IPC-only channel reply method called by the gateway over the assistant IPC
+ * socket (`ipcCallAssistant`).
+ *
+ * The gateway answers some inbound messages itself without forwarding them
+ * (a verification code, an invite redemption), so the assistant never sees
+ * them. Its reply still has to reach the person, and the channel transports
+ * that can send it live here. This hands the reply to the transport the
+ * inbound message's callback URL names, which is how the daemon's own
+ * intercepts answer an inbound message: the gateway sends through the same
+ * transports and never to a provider itself.
+ *
+ * Nothing is recorded in a conversation. The message the reply answers never
+ * entered one, so a row for the reply would stand alone in a transcript that
+ * does not hold what it responds to.
+ */
+
+import {
+  DELIVER_GATEWAY_REPLY_IPC_METHOD,
+  GatewayReplyRequestSchema,
+} from "@vellumai/gateway-client";
+
+import {
+  deliverDirect,
+  isDirectDelivery,
+} from "../../messaging/providers/index.js";
+import {
+  BadGatewayError,
+  BadRequestError,
+} from "../../runtime/routes/errors.js";
+import { parseBody } from "../../runtime/routes/parse-body.js";
+import type { RouteHandlerArgs } from "../../runtime/routes/types.js";
+
+/**
+ * Deliver a gateway-composed reply through the channel transport its callback
+ * URL names. Refuses a callback no transport owns rather than fetching it, so
+ * the gateway learns the channel cannot be answered this way.
+ *
+ * A send the channel refuses is answered as a `BadGatewayError`, as
+ * `channels/send` answers it: only a `RouteError` carries a status over IPC,
+ * and the gateway reads an error without one as a daemon that never answered.
+ */
+export async function handleDeliverGatewayReply({
+  body = {},
+}: RouteHandlerArgs) {
+  const { callbackUrl, chatId, text, assistantId } = parseBody(
+    GatewayReplyRequestSchema,
+    body,
+  );
+  if (!isDirectDelivery(callbackUrl)) {
+    throw new BadRequestError("No channel transport owns this callback URL");
+  }
+  try {
+    return await deliverDirect(callbackUrl, {
+      chatId,
+      text,
+      ...(assistantId ? { assistantId } : {}),
+    });
+  } catch (err) {
+    throw new BadGatewayError(err instanceof Error ? err.message : String(err));
+  }
+}
+
+/**
+ * IPC-only channel reply methods, keyed by IPC operationId. Registered
+ * directly on the assistant IPC server (see `assistant-server.ts`).
+ */
+export const CHANNEL_REPLY_IPC_METHODS: Record<
+  string,
+  (args: RouteHandlerArgs) => unknown
+> = {
+  [DELIVER_GATEWAY_REPLY_IPC_METHOD]: handleDeliverGatewayReply,
+};

package/src/live-voice/__tests__/continuation-judge.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, test } from "bun:test";
+
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+} from "../../providers/types.js";
+import {
+  CONTINUATION_JUDGE_KEEP_THRESHOLD,
+  createContinuationJudge,
+  VOICE_CONTINUATION_JUDGE_CALL_SITE,
+} from "../continuation-judge.js";
+
+function jevResponse(noul: number): ProviderResponse {
+  const answers = { answer: { type: "noul", noul } };
+  return {
+    content: [{ type: "text", text: JSON.stringify(answers) }],
+    model: "jev-latest",
+    usage: { inputTokens: 1, outputTokens: 1 },
+    stopReason: "end_turn",
+    rawResponse: { model: "jev-latest", answers },
+  };
+}
+
+function fakeProvider(
+  respond: (
+    messages: Message[],
+    options?: SendMessageOptions,
+  ) => Promise<ProviderResponse>,
+): Provider {
+  return { name: "typesafe", sendMessage: respond } as Provider;
+}
+
+const baseArgs = {
+  parentConversationId: "conv-1",
+  interruptedRequest: "Can you hit join now for me?",
+  signal: new AbortController().signal,
+};
+
+describe("createContinuationJudge", () => {
+  test("drops below the threshold and asks with both utterances", async () => {
+    let seen: { body: unknown; options?: SendMessageOptions } | null = null;
+    const judge = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async (messages, options) => {
+          const block = messages[0]!.content[0]!;
+          seen = {
+            body: JSON.parse(block.type === "text" ? block.text : ""),
+            options,
+          };
+          return jevResponse(0.09);
+        }),
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      interruption: Promise.resolve("Actually just point it out for me."),
+    });
+
+    expect(judgement).toMatchObject({
+      keep: false,
+      outcome: "drop",
+      noul: 0.09,
+    });
+    const request = seen as unknown as {
+      body: { state: Record<string, string> };
+      options: SendMessageOptions;
+    };
+    expect(request.body.state).toEqual({
+      interrupted_request: "Can you hit join now for me?",
+      caller_then_said: "Actually just point it out for me.",
+    });
+    expect(
+      (request.options.config as { callSite?: string } | undefined)?.callSite,
+    ).toBe(VOICE_CONTINUATION_JUDGE_CALL_SITE);
+  });
+
+  test("keeps at or above the threshold", async () => {
+    const judge = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async () =>
+          jevResponse(CONTINUATION_JUDGE_KEEP_THRESHOLD),
+        ),
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      interruption: Promise.resolve("Oh, also, what's the weather?"),
+    });
+
+    expect(judgement).toMatchObject({ keep: true, outcome: "keep" });
+  });
+
+  test("without a TypeSafe provider it keeps and never waits on the interruption", async () => {
+    const judge = createContinuationJudge({
+      resolveProvider: async () => null,
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      // Never settles: the judge must not await it.
+      interruption: new Promise<string | null>(() => {}),
+    });
+
+    expect(judgement).toMatchObject({ keep: true, outcome: "unavailable" });
+  });
+
+  test("keeps when no interruption arrived in time", async () => {
+    let asked = false;
+    const judge = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async () => {
+          asked = true;
+          return jevResponse(0.05);
+        }),
+    });
+
+    const judgement = await judge({
+      ...baseArgs,
+      interruption: Promise.resolve(null),
+    });
+
+    expect(judgement).toMatchObject({
+      keep: true,
+      outcome: "no_interruption",
+    });
+    expect(asked).toBe(false);
+  });
+
+  test("a provider error or timeout keeps the continuation", async () => {
+    const failing = createContinuationJudge({
+      resolveProvider: async () =>
+        fakeProvider(async () => {
+          throw new Error("503");
+        }),
+    });
+    const hanging = createContinuationJudge({
+      timeoutMs: 20,
+      resolveProvider: async () =>
+        fakeProvider(async () => new Promise<ProviderResponse>(() => {})),
+    });
+
+    expect(
+      await failing({ ...baseArgs, interruption: Promise.resolve("stop") }),
+    ).toMatchObject({ keep: true, outcome: "error" });
+    expect(
+      await hanging({ ...baseArgs, interruption: Promise.resolve("stop") }),
+    ).toMatchObject({ keep: true, outcome: "timeout" });
+  });
+});

package/src/live-voice/__tests__/live-activity-reporter.test.ts

@@ -37,6 +37,10 @@ class RecordingReporter extends LiveActivityReporter {
   readonly dispatched: Array<{ phase: string; event: string; detail: string }> =
     [];
 
+  waitForDispatches(): Promise<void> {
+    return this.waitForPendingDispatches();
+  }
+
   protected override async dispatch(
     phase: string,
     event: "update" | "end",
@@ -46,11 +50,55 @@ class RecordingReporter extends LiveActivityReporter {
   }
 }
 
-function activity(label: string): LiveVoiceServerFramePayload {
+class BlockingReporter extends LiveActivityReporter {
+  readonly dispatched: string[] = [];
+  private releaseFirstDispatch: (() => void) | null = null;
+  private readonly firstDispatch = new Promise<void>((resolve) => {
+    this.releaseFirstDispatch = resolve;
+  });
+
+  releaseFirst(): void {
+    this.releaseFirstDispatch?.();
+  }
+
+  waitForDispatches(): Promise<void> {
+    return this.waitForPendingDispatches();
+  }
+
+  protected override async dispatch(phase: string): Promise<void> {
+    this.dispatched.push(phase);
+    if (this.dispatched.length === 1) {
+      await this.firstDispatch;
+    }
+  }
+}
+
+class TimeoutReporter extends LiveActivityReporter {
+  readonly dispatched: string[] = [];
+  protected override readonly dispatchTimeoutMs = 1;
+
+  waitForDispatches(): Promise<void> {
+    return this.waitForPendingDispatches();
+  }
+
+  protected override async dispatch(phase: string): Promise<void> {
+    this.dispatched.push(phase);
+    if (this.dispatched.length > 1) {
+      return;
+    }
+    await new Promise<void>(() => undefined);
+  }
+}
+
+function activity(
+  label: string,
+  kind?: "escalation",
+): LiveVoiceServerFramePayload {
   return {
     type: "activity",
     turnId: "t1",
     label,
+    ...(kind ? { kind } : {}),
   } as LiveVoiceServerFramePayload;
 }
 
@@ -61,6 +109,9 @@ describe("phaseForFrame", () => {
     );
     expect(phaseForFrame(frame("thinking"), "transcribing")).toBe("thinking");
     expect(phaseForFrame(frame("tts_audio"), "thinking")).toBe("speaking");
+    expect(
+      phaseForFrame(activity("Working on that", "escalation"), "speaking"),
+    ).toBe("thinking");
   });
 
   test("a discarded utterance returns the floor to the user", () => {
@@ -96,10 +147,11 @@ describe("phaseForFrame", () => {
 });
 
 describe("LiveActivityReporter", () => {
-  test("reports a phase change once", () => {
+  test("reports a phase change once", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("thinking"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toEqual([
       { phase: "thinking", event: "update", detail: "" },
@@ -107,26 +159,31 @@ describe("LiveActivityReporter", () => {
   });
 
   // The whole reason this class exists rather than a bare fetch per frame.
-  test("does not re-report a phase it is already in", () => {
+  test("does not re-report a phase it is already in", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("tts_audio"));
     reporter.report(frame("tts_audio"));
     reporter.report(frame("tts_audio"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toEqual([
       { phase: "speaking", event: "update", detail: "" },
     ]);
   });
 
-  test("reports each genuine transition through a turn", () => {
+  test("reports each genuine transition through a turn", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("utterance_end"));
+    await reporter.waitForDispatches();
     reporter.report(sttFinal("hello there"));
+    await reporter.waitForDispatches();
     reporter.report(frame("tts_audio"));
     reporter.report(frame("tts_audio"));
+    await reporter.waitForDispatches();
     reporter.report(frame("tts_done"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.map((d) => d.phase)).toEqual([
       "transcribing",
@@ -136,20 +193,63 @@ describe("LiveActivityReporter", () => {
     ]);
   });
 
-  test("frames that do not move the phase are ignored", () => {
+  test("serializes platform dispatches in frame order", async () => {
+    const reporter = new BlockingReporter("conv-1");
+
+    reporter.report(frame("tts_audio"));
+    reporter.report(activity("", "escalation"));
+    await Promise.resolve();
+
+    expect(reporter.dispatched).toEqual(["speaking"]);
+
+    reporter.releaseFirst();
+    await reporter.waitForDispatches();
+    expect(reporter.dispatched).toEqual(["speaking", "thinking"]);
+  });
+
+  test("a timed-out dispatch does not pin later phases", async () => {
+    const reporter = new TimeoutReporter("conv-1");
+
+    reporter.report(frame("tts_audio"));
+    reporter.report(activity("", "escalation"));
+    await reporter.waitForDispatches();
+
+    expect(reporter.dispatched).toEqual(["speaking", "thinking"]);
+  });
+
+  test("coalesces a queued backlog and keeps end next", async () => {
+    const reporter = new BlockingReporter("conv-1");
+
+    reporter.report(frame("utterance_end"));
+    reporter.report(frame("thinking"));
+    reporter.report(activity("Running a command"));
+    reporter.report(frame("tts_audio"));
+    reporter.end();
+    await Promise.resolve();
+
+    expect(reporter.dispatched).toEqual(["transcribing"]);
+
+    reporter.releaseFirst();
+    await reporter.waitForDispatches();
+    expect(reporter.dispatched).toEqual(["transcribing", "ending"]);
+  });
+
+  test("frames that do not move the phase are ignored", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("stt_partial"));
     reporter.report(frame("ready"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toEqual([]);
   });
 
-  test("ending retires the activity", () => {
+  test("ending retires the activity", async () => {
     const reporter = new RecordingReporter("conv-1");
     reporter.report(frame("tts_audio"));
 
     reporter.end();
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.at(-1)).toEqual({
       phase: "ending",
@@ -161,20 +261,22 @@ describe("LiveActivityReporter", () => {
   // A session can end more than one way and the platform deletes the
   // registration on the first, so a second end would push at an activity that
   // is already gone.
-  test("ending twice pushes once", () => {
+  test("ending twice pushes once", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.end();
     reporter.end();
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toHaveLength(1);
   });
 
-  test("a frame after the end is not reported", () => {
+  test("a frame after the end is not reported", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.end();
     reporter.report(frame("tts_audio"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toEqual([
       { phase: "ending", event: "end", detail: "" },
@@ -183,12 +285,26 @@ describe("LiveActivityReporter", () => {
 });
 
 describe("LiveActivityReporter activity line", () => {
-  test("carries the activity label alongside the phase it holds", () => {
+  test("keeps escalation out of the Live Activity detail line", async () => {
+    const reporter = new RecordingReporter("conv-1");
+
+    reporter.report(frame("tts_audio"));
+    reporter.report(activity("Working on that", "escalation"));
+    await reporter.waitForDispatches();
+
+    expect(reporter.dispatched).toEqual([
+      { phase: "speaking", event: "update", detail: "" },
+      { phase: "thinking", event: "update", detail: "" },
+    ]);
+  });
+
+  test("carries the activity label alongside the phase it holds", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("utterance_end"));
     reporter.report(frame("thinking"));
     reporter.report(activity("Running a command"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.at(-1)).toEqual({
       phase: "thinking",
@@ -199,12 +315,13 @@ describe("LiveActivityReporter activity line", () => {
 
   // A push replaces the whole content state, so the label has to ride along on
   // phase changes too, or the next phase would blank it.
-  test("keeps the label through a later phase change", () => {
+  test("keeps the label through a later phase change", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("thinking"));
     reporter.report(activity("Reading a file"));
     reporter.report(frame("tts_audio"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.at(-1)).toEqual({
       phase: "speaking",
@@ -213,36 +330,56 @@ describe("LiveActivityReporter activity line", () => {
     });
   });
 
-  test("an empty label clears the line", () => {
+  test("restores thinking after bridge speech without clearing an overlay", async () => {
+    const reporter = new RecordingReporter("conv-1");
+
+    reporter.report(frame("thinking"));
+    reporter.report(frame("tts_audio"));
+    reporter.report(activity("Searching the web"));
+    reporter.restoreThinkingPhase();
+    await reporter.waitForDispatches();
+
+    expect(reporter.dispatched.at(-1)).toEqual({
+      phase: "thinking",
+      event: "update",
+      detail: "Searching the web",
+    });
+  });
+
+  test("an empty label clears the line", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("thinking"));
     reporter.report(activity("Reading a file"));
     reporter.report(activity(""));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.at(-1)?.detail).toBe("");
   });
 
-  test("does not dispatch a label it already sent", () => {
+  test("does not dispatch a label it already sent", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("thinking"));
     reporter.report(activity("Reading a file"));
+    await reporter.waitForDispatches();
     const before = reporter.dispatched.length;
     reporter.report(activity("Reading a file"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.length).toBe(before);
   });
 
   // A tool can start before any phase-bearing frame lands. There is no content
   // state to attach a label to yet, and the phase that follows carries it.
-  test("holds a label that arrives before any phase", () => {
+  test("holds a label that arrives before any phase", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(activity("Searching the web"));
     expect(reporter.dispatched).toEqual([]);
 
     reporter.report(frame("thinking"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toEqual([
       { phase: "thinking", event: "update", detail: "Searching the web" },
@@ -253,19 +390,21 @@ describe("LiveActivityReporter activity line", () => {
 describe("LiveActivityReporter and held utterances", () => {
   // End to end through the reporter, not just the mapping: a session using
   // semantic endpointing emits finals while the user still holds the floor.
-  test("a final during a held utterance pushes nothing", () => {
+  test("a final during a held utterance pushes nothing", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(sttFinal("i was thinking maybe"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched).toEqual([]);
   });
 
-  test("the same final after utterance_end advances to thinking", () => {
+  test("the same final after utterance_end advances to thinking", async () => {
     const reporter = new RecordingReporter("conv-1");
 
     reporter.report(frame("utterance_end"));
     reporter.report(sttFinal("i was thinking maybe"));
+    await reporter.waitForDispatches();
 
     expect(reporter.dispatched.map((d) => d.phase)).toEqual([
       "transcribing",