@vellumai/assistant 0.11.5 → 0.11.6-staging.1

package/src/runtime/routes/inbound-stages/background-dispatch.test.ts

@@ -88,18 +88,29 @@ mock.module("../../gateway-client.js", () => ({
   },
 }));
 
-const sentReactions: Array<{
-  callbackUrl: string;
-  target: Record<string, unknown>;
-}> = [];
 const sentStreamOps: Array<Record<string, unknown>> = [];
 let sendChannelStreamOpImpl: (
   op: Record<string, unknown>,
 ) => Promise<{ ok: boolean; ts?: string }> = async () => ({ ok: true });
-const sentThreadStatuses: Array<Record<string, unknown>> = [];
+const sentActivity: Array<Record<string, unknown>> = [];
+let setChannelActivityImpl: (
+  target: Record<string, unknown>,
+) => { ok: boolean } | Promise<{ ok: boolean }> = () => ({ ok: true });
+// Undefined stands for a channel whose indicator holds until it is changed, so
+// the controller reports each phase once and the assertions read as the turn's
+// lifecycle rather than as timer ticks. A test that needs the self-expiring
+// shape sets a cadence.
+let activityRefreshMsImpl: () => number | undefined = () => undefined;
 mock.module("../../../messaging/providers/index.js", () => ({
-  sendChannelTyping: async () => ({ ok: true }),
-  supportsChannelTyping: () => false,
+  supportsChannelActivity: () => true,
+  channelActivityRefreshMs: () => activityRefreshMsImpl(),
+  setChannelActivity: async (
+    _callbackUrl: string,
+    target: Record<string, unknown>,
+  ) => {
+    sentActivity.push(target);
+    return setChannelActivityImpl(target);
+  },
   sendChannelStreamOp: async (
     _callbackUrl: string,
     _chatId: string,
@@ -108,20 +119,6 @@ mock.module("../../../messaging/providers/index.js", () => ({
     sentStreamOps.push(op);
     return sendChannelStreamOpImpl(op);
   },
-  setChannelThreadStatus: async (
-    _callbackUrl: string,
-    status: Record<string, unknown>,
-  ) => {
-    sentThreadStatuses.push(status);
-    return { ok: true };
-  },
-  sendChannelReaction: async (
-    callbackUrl: string,
-    target: Record<string, unknown>,
-  ) => {
-    sentReactions.push({ callbackUrl, target });
-    return { ok: true };
-  },
 }));
 
 mock.module("../../channel-reply-delivery.js", () => ({
@@ -154,8 +151,8 @@ import type { MessageProcessor } from "../../http-types.js";
 import {
   isBoundGuardianActor,
   processChannelMessageInBackground,
-  shouldStartSlackThinkingStatusForText,
-  shouldStartSlackThinkingStatusImmediately,
+  shouldShowActivityForText,
+  shouldShowActivityImmediately,
 } from "./background-dispatch.js";
 import { __resetChannelTurnAdmissionForTests } from "./channel-turn-admission.js";
 
@@ -163,10 +160,11 @@ beforeEach(() => {
   __resetChannelTurnAdmissionForTests();
   clearConversations();
   deliveredChannelReplies.length = 0;
-  sentReactions.length = 0;
   sentStreamOps.length = 0;
   sendChannelStreamOpImpl = async () => ({ ok: true });
-  sentThreadStatuses.length = 0;
+  sentActivity.length = 0;
+  setChannelActivityImpl = () => ({ ok: true });
+  activityRefreshMsImpl = () => undefined;
   markedProcessedEvents.length = 0;
   processingFailureEvents.length = 0;
   retryableFailureEvents.length = 0;
@@ -568,11 +566,11 @@ describe("processChannelMessageInBackground — reply delivery", () => {
     expect(slackStreamOps()).toEqual([
       {
         action: "start",
-        threadTs,
-        markdownText: "Streamed DM reply.",
-        taskDisplayMode: "plan",
+        anchorMessageId: threadTs,
+        text: "Streamed DM reply.",
+        appended: "Streamed DM reply.",
       },
-      { action: "stop", streamTs },
+      { action: "stop", streamId: streamTs, text: "Streamed DM reply." },
     ]);
     expect(
       deliveredChannelReplies
@@ -893,13 +891,7 @@ describe("processChannelMessageInBackground — admission (queue if busy)", () =
   });
 });
 
-describe("Slack thinking status timing", () => {
-  const slackStatusLabels = [
-    "is on it",
-    "is working hard",
-    "is touching grass",
-  ];
-
+describe("channel activity timing", () => {
   const trustCtx: TrustContext = {
     trustClass: "guardian",
     guardianExternalUserId: "guardian-1",
@@ -909,73 +901,74 @@ describe("Slack thinking status timing", () => {
   const flush = (): Promise<void> =>
     new Promise((resolve) => setTimeout(resolve, 10));
 
+  const phases = (): unknown[] =>
+    sentActivity.map((entry) => (entry as { phase?: unknown }).phase);
+
   beforeEach(() => {
     deliveredChannelReplies.length = 0;
   });
 
-  test("recognizes only deliverable text as a Slack thinking-status trigger", () => {
-    expect(shouldStartSlackThinkingStatusForText("")).toBe(false);
-    expect(shouldStartSlackThinkingStatusForText("   ")).toBe(false);
-    expect(shouldStartSlackThinkingStatusForText("<")).toBe(false);
-    expect(shouldStartSlackThinkingStatusForText("<no_response")).toBe(false);
-    expect(shouldStartSlackThinkingStatusForText("<no_response/>")).toBe(false);
-    expect(shouldStartSlackThinkingStatusForText("  <no_response />  ")).toBe(
+  test("recognizes only deliverable text as a reason to show activity", () => {
+    expect(shouldShowActivityForText("")).toBe(false);
+    expect(shouldShowActivityForText("   ")).toBe(false);
+    expect(shouldShowActivityForText("<")).toBe(false);
+    expect(shouldShowActivityForText("<no_response")).toBe(false);
+    expect(shouldShowActivityForText("<no_response/>")).toBe(false);
+    expect(shouldShowActivityForText("  <no_response />  ")).toBe(false);
+    expect(shouldShowActivityForText("Real response.")).toBe(true);
+    expect(shouldShowActivityForText("<no_response/>\nReal response.")).toBe(
+      true,
+    );
+  });
+
+  test("recognizes a direct conversation in every channel's word for one", () => {
+    // One idea, three spellings, because `chatType` arrives as whatever the
+    // channel called it. Slack `im`, Discord `dm`, Telegram `private`. Missing
+    // one does not fail loudly: that channel silently reads as an ambient room
+    // and stops showing an indicator it used to show.
+    expect(shouldShowActivityImmediately({ chatType: "im" })).toBe(true);
+    expect(shouldShowActivityImmediately({ chatType: "dm" })).toBe(true);
+    expect(shouldShowActivityImmediately({ chatType: "private" })).toBe(true);
+  });
+
+  test("treats a room with other readers as ambient, including a group DM", () => {
+    // Slack's `mpim` has other readers, so it is a room: the assistant may
+    // process the traffic and decide to stay quiet, and an indicator there
+    // announces it was reading.
+    expect(shouldShowActivityImmediately({ chatType: "mpim" })).toBe(false);
+    expect(shouldShowActivityImmediately({ chatType: "channel" })).toBe(false);
+    expect(shouldShowActivityImmediately({ chatType: "supergroup" })).toBe(
       false,
     );
-    expect(shouldStartSlackThinkingStatusForText("Real response.")).toBe(true);
-    expect(
-      shouldStartSlackThinkingStatusForText("<no_response/>\nReal response."),
-    ).toBe(true);
+    expect(shouldShowActivityImmediately({})).toBe(false);
   });
 
-  test("starts Slack thinking status immediately for DMs and direct mentions", () => {
-    expect(
-      shouldStartSlackThinkingStatusImmediately({
-        sourceChannel: "slack",
-        chatType: "im",
-      }),
-    ).toBe(true);
-    expect(
-      shouldStartSlackThinkingStatusImmediately({
-        sourceChannel: "slack",
-        slackBotMentioned: true,
-      }),
-    ).toBe(true);
+  test("shows activity immediately when the assistant is mentioned", () => {
+    expect(shouldShowActivityImmediately({ botMentioned: true })).toBe(true);
     expect(
-      shouldStartSlackThinkingStatusImmediately({
-        sourceChannel: "slack",
+      shouldShowActivityImmediately({
         chatType: "channel",
+        botMentioned: true,
       }),
-    ).toBe(false);
-    expect(
-      shouldStartSlackThinkingStatusImmediately({
-        sourceChannel: "telegram",
-        chatType: "im",
-        slackBotMentioned: true,
-      }),
-    ).toBe(false);
+    ).toBe(true);
   });
 
-  test("sets Slack thinking indicator immediately for a DM", async () => {
-    const conversationId = "conv-dm-immediate-status";
+  test("shows activity immediately in a DM and settles it when the turn ends", async () => {
+    const conversationId = "conv-dm-immediate-activity";
     const channelId = "D-DM-IMMEDIATE";
     const messageTs = "1700000000.000010";
 
     const processMessage: MessageProcessor = async () => {
-      expect(sentReactions).toHaveLength(1);
-      expect(sentReactions[0]!.target).toEqual({
-        chatId: channelId,
-        messageId: messageTs,
-        emoji: "eyes",
-        action: "add",
-      });
+      // Already showing before the turn does any work: a DM is addressed to
+      // the assistant, so nothing needs to be observed first.
+      expect(phases()).toEqual(["thinking"]);
       return { messageId: "user-msg-dm-immediate" };
     };
 
     processChannelMessageInBackground({
       processMessage,
       conversationId,
-      eventId: "evt-dm-immediate-status",
+      eventId: "evt-dm-immediate-activity",
       content: "dm message",
       sourceChannel: "slack",
       sourceInterface: "slack",
@@ -988,39 +981,27 @@ describe("Slack thinking status timing", () => {
 
     await flush();
 
-    expect(sentReactions.map((entry) => entry.target)).toEqual([
-      { chatId: channelId, messageId: messageTs, emoji: "eyes", action: "add" },
-      {
-        chatId: channelId,
-        messageId: messageTs,
-        emoji: "eyes",
-        action: "remove",
-      },
-    ]);
+    // `idle` is owed explicitly. A channel that holds its indicator keeps it
+    // up until told otherwise, so a missing terminal phase is a stuck spinner
+    // rather than a cosmetic slip.
+    expect(phases()).toEqual(["thinking", "idle"]);
+    expect(sentActivity[0]).toEqual({ chatId: channelId, phase: "thinking" });
   });
 
-  test("sets Slack thinking status immediately for an app mention", async () => {
-    const conversationId = "conv-mention-immediate-status";
+  test("shows activity immediately for an app mention and settles it", async () => {
+    const conversationId = "conv-mention-immediate-activity";
     const channelId = "C-MENTION-IMMEDIATE";
     const threadTs = "1700000000.000011";
 
     const processMessage: MessageProcessor = async () => {
-      expect(sentThreadStatuses).toHaveLength(1);
-      expect(sentThreadStatuses[0]).toEqual({
-        chatId: channelId,
-        threadTs,
-        status: expect.any(String),
-        loadingMessages: ["Thinking\u2026"],
-      });
-      const threadStatus = sentThreadStatuses[0] as { status: string };
-      expect(slackStatusLabels).toContain(threadStatus.status);
+      expect(phases()).toEqual(["thinking"]);
       return { messageId: "user-msg-mention-immediate" };
     };
 
     processChannelMessageInBackground({
       processMessage,
       conversationId,
-      eventId: "evt-mention-immediate-status",
+      eventId: "evt-mention-immediate-activity",
       content: "@assistant please respond",
       sourceChannel: "slack",
       sourceInterface: "slack",
@@ -1033,204 +1014,123 @@ describe("Slack thinking status timing", () => {
 
     await flush();
 
-    const statuses = sentThreadStatuses.map(
-      (entry) => (entry as { status?: string }).status,
-    );
-    expect(slackStatusLabels).toContain(statuses[0]!);
-    expect(statuses[1]).toBe("");
+    expect(phases()).toEqual(["thinking", "idle"]);
   });
 
-  test("does not set Slack thinking status for no_response text deltas", async () => {
-    const conversationId = "conv-no-response-status";
-    const channelId = "C-NO-RESPONSE";
-    const threadTs = "1700000000.000003";
-
-    const processMessage: MessageProcessor = async (
-      _conversationId,
-      _content,
-      options,
-    ) => {
-      options?.onEvent?.({
-        type: "assistant_text_delta",
-        text: "<no_response/>",
-        conversationId,
-      });
-      return { messageId: "user-msg-no-response" };
+  test("retries a terminal transition the channel failed to accept", async () => {
+    const conversationId = "conv-idle-retry";
+    const channelId = "D-IDLE-RETRY";
+    let idleAttempts = 0;
+    setChannelActivityImpl = (target) => {
+      if (target.phase === "idle") {
+        idleAttempts += 1;
+        return { ok: idleAttempts > 1 };
+      }
+      return { ok: true };
     };
 
     processChannelMessageInBackground({
-      processMessage,
+      processMessage: async () => ({ messageId: "user-msg-idle-retry" }),
       conversationId,
-      eventId: "evt-no-response-status",
-      content: "ambient channel chatter",
+      eventId: "evt-idle-retry",
+      content: "dm message",
       sourceChannel: "slack",
       sourceInterface: "slack",
       externalChatId: channelId,
       trustCtx,
       metadataHints: [],
-      replyCallbackUrl: `https://example.test/deliver/slack?channel=${channelId}&threadTs=${threadTs}`,
+      chatType: "im",
+      replyCallbackUrl: `https://example.test/deliver/slack?channel=${channelId}&messageTs=1700000000.000020`,
     });
 
     await flush();
 
-    expect(deliveredChannelReplies).toEqual([]);
+    // A dropped `idle` is not cosmetic on a channel that holds its indicator:
+    // Slack keeps showing the assistant as working for an hour, so the
+    // terminal transition is the one worth attempting twice.
+    expect(idleAttempts).toBe(2);
   });
 
-  test("sets and clears Slack thinking status after real assistant text starts", async () => {
-    const conversationId = "conv-real-response-status";
-    const channelId = "C-REAL-RESPONSE";
-    const threadTs = "1700000000.000004";
-
-    const processMessage: MessageProcessor = async (
-      _conversationId,
-      _content,
-      options,
-    ) => {
-      options?.onEvent?.({
-        type: "assistant_text_delta",
-        text: "<",
-        conversationId,
-      });
-      expect(deliveredChannelReplies).toEqual([]);
-
-      options?.onEvent?.({
-        type: "assistant_text_delta",
-        text: "b>Working on it.",
-        conversationId,
-      });
-      return { messageId: "user-msg-real-response" };
+  test("runs idle next when the turn stops during a slow request, with no stale busy phase between", async () => {
+    const conversationId = "conv-slow-refresh";
+    const channelId = "D-SLOW-REFRESH";
+
+    // A self-expiring channel re-asserts on a timer, which is the only shape
+    // where a refresh backlog can form at all.
+    activityRefreshMsImpl = () => 5;
+
+    let release: (() => void) | undefined;
+    let calls = 0;
+    setChannelActivityImpl = (target) => {
+      calls += 1;
+      if (calls === 1 && target.phase !== "idle") {
+        // Hold the first busy call open across many refresh intervals.
+        return new Promise<{ ok: boolean }>((resolve) => {
+          release = () => resolve({ ok: true });
+        });
+      }
+      return { ok: true };
     };
 
-    processChannelMessageInBackground({
-      processMessage,
-      conversationId,
-      eventId: "evt-real-response-status",
-      content: "please respond",
-      sourceChannel: "slack",
-      sourceInterface: "slack",
-      externalChatId: channelId,
-      trustCtx,
-      metadataHints: [],
-      replyCallbackUrl: `https://example.test/deliver/slack?channel=${channelId}&threadTs=${threadTs}`,
-    });
-
-    await flush();
-
-    const statuses = sentThreadStatuses.map(
-      (entry) => (entry as { status?: string }).status,
-    );
-    expect(slackStatusLabels).toContain(statuses[0]!);
-    expect(statuses[1]).toBe("");
-  });
-
-  test("buffers task_progress for ambiguous Slack turns until deliverable text appears", async () => {
-    const conversationId = "conv-progress-buffered";
-    const channelId = "C-PROGRESS-BUFFERED";
-    const threadTs = "1700000000.000012";
-
-    const processMessage: MessageProcessor = async (
-      _conversationId,
-      _content,
-      options,
-    ) => {
-      options?.onEvent?.({
-        type: "ui_surface_show",
-        conversationId,
-        surfaceId: "surface-progress",
-        surfaceType: "card",
-        data: {
-          title: "Task progress",
-          body: "Working",
-          template: "task_progress",
-          templateData: {
-            steps: [
-              { label: "Search docs", status: "in_progress" },
-              { label: "Summarize", status: "pending" },
-            ],
-          },
-        },
-      });
-      expect(deliveredChannelReplies).toEqual([]);
-
-      options?.onEvent?.({
-        type: "assistant_text_delta",
-        text: "I found the answer.",
-        conversationId,
-      });
-      return { messageId: "user-msg-progress-buffered" };
+    const processMessage: MessageProcessor = async () => {
+      // Long enough for several ticks to fire while the first call is unresolved.
+      await new Promise((resolve) => setTimeout(resolve, 40));
+      return { messageId: "user-msg-slow-refresh" };
     };
 
     processChannelMessageInBackground({
       processMessage,
       conversationId,
-      eventId: "evt-progress-buffered",
-      content: "ambient request",
+      eventId: "evt-slow-refresh",
+      content: "dm message",
       sourceChannel: "slack",
       sourceInterface: "slack",
       externalChatId: channelId,
       trustCtx,
       metadataHints: [],
-      replyCallbackUrl: `https://example.test/deliver/slack?channel=${channelId}&threadTs=${threadTs}`,
+      chatType: "im",
+      replyCallbackUrl: `https://example.test/deliver/slack?channel=${channelId}&messageTs=1700000000.000030`,
     });
 
+    await new Promise((resolve) => setTimeout(resolve, 60));
+    release?.();
     await flush();
 
-    const statuses = sentThreadStatuses;
-    expect(statuses).toEqual([
-      {
-        chatId: channelId,
-        threadTs,
-        status: expect.any(String),
-        loadingMessages: ["In progress (1/2): Search docs"],
-      },
-      {
-        chatId: channelId,
-        threadTs,
-        status: "",
-      },
-    ]);
-    expect(slackStatusLabels).toContain(
-      (statuses[0] as { status: string }).status,
-    );
+    const seen = phases();
+    // The terminal phase is last and nothing busy follows it. A busy phase
+    // executing after `idle` is the visible bug: the indicator comes back on a
+    // turn that has already replied, which teaches people to distrust it.
+    expect(seen.at(-1)).toBe("idle");
+    expect(seen.slice(seen.indexOf("idle") + 1)).toEqual([]);
+    // Ticks did not accumulate behind the outstanding call. Counting rather
+    // than checking the last entry is what catches a backlog that happens to
+    // drain in a harmless order.
+    expect(seen.filter((phase) => phase === "thinking")).toHaveLength(1);
   });
 
-  test("keeps ambiguous Slack no_response turns quiet even with task_progress", async () => {
-    const conversationId = "conv-progress-no-response";
-    const channelId = "C-PROGRESS-NO-RESPONSE";
-    const threadTs = "1700000000.000013";
+  test("stays silent for a turn that decides not to answer", async () => {
+    const conversationId = "conv-no-response-activity";
+    const channelId = "C-NO-RESPONSE";
+    const threadTs = "1700000000.000003";
 
     const processMessage: MessageProcessor = async (
       _conversationId,
       _content,
       options,
     ) => {
-      options?.onEvent?.({
-        type: "ui_surface_show",
-        conversationId,
-        surfaceId: "surface-progress-no-response",
-        surfaceType: "card",
-        data: {
-          title: "Task progress",
-          body: "Working",
-          template: "task_progress",
-          templateData: {
-            steps: [{ label: "Inspect", status: "in_progress" }],
-          },
-        },
-      });
       options?.onEvent?.({
         type: "assistant_text_delta",
         text: "<no_response/>",
         conversationId,
       });
-      return { messageId: "user-msg-progress-no-response" };
+      return { messageId: "user-msg-no-response" };
     };
 
     processChannelMessageInBackground({
       processMessage,
       conversationId,
-      eventId: "evt-progress-no-response",
-      content: "ambient chatter",
+      eventId: "evt-no-response-activity",
+      content: "ambient channel chatter",
       sourceChannel: "slack",
       sourceInterface: "slack",
       externalChatId: channelId,
@@ -1241,61 +1141,44 @@ describe("Slack thinking status timing", () => {
 
     await flush();
 
+    // Not merely "no spinner": nothing at all was said to the channel. An
+    // `idle` here would still be a message about a turn nobody asked to see,
+    // and in a shared room it announces that the assistant read the traffic.
+    expect(sentActivity).toEqual([]);
     expect(deliveredChannelReplies).toEqual([]);
   });
 
-  test("updates Slack loading message when task_progress changes", async () => {
-    const conversationId = "conv-progress-update";
-    const channelId = "C-PROGRESS-UPDATE";
-    const threadTs = "1700000000.000014";
+  test("waits for real text in a room it was not addressed in, then settles", async () => {
+    const conversationId = "conv-real-response-activity";
+    const channelId = "C-REAL-RESPONSE";
+    const threadTs = "1700000000.000004";
 
     const processMessage: MessageProcessor = async (
       _conversationId,
       _content,
       options,
     ) => {
-      options?.onEvent?.({
-        type: "ui_surface_show",
-        conversationId,
-        surfaceId: "surface-progress-update",
-        surfaceType: "card",
-        data: {
-          title: "Task progress",
-          body: "Working",
-          template: "task_progress",
-          templateData: {
-            steps: [
-              { label: "Read request", status: "in_progress" },
-              { label: "Write answer", status: "pending" },
-            ],
-          },
-        },
-      });
       options?.onEvent?.({
         type: "assistant_text_delta",
-        text: "On it.",
+        text: "<",
         conversationId,
       });
+      // A partial `<` could still become `<no_response/>`, so nothing shows.
+      expect(sentActivity).toEqual([]);
+
       options?.onEvent?.({
-        type: "ui_surface_update",
+        type: "assistant_text_delta",
+        text: "b>Working on it.",
         conversationId,
-        surfaceId: "surface-progress-update",
-        data: {
-          templateData: {
-            steps: [
-              { label: "Read request", status: "completed" },
-              { label: "Write answer", status: "in_progress" },
-            ],
-          },
-        },
       });
-      return { messageId: "user-msg-progress-update" };
+      expect(phases()).toEqual(["thinking"]);
+      return { messageId: "user-msg-real-response" };
     };
 
     processChannelMessageInBackground({
       processMessage,
       conversationId,
-      eventId: "evt-progress-update",
+      eventId: "evt-real-response-activity",
       content: "please respond",
       sourceChannel: "slack",
       sourceInterface: "slack",
@@ -1307,31 +1190,6 @@ describe("Slack thinking status timing", () => {
 
     await flush();
 
-    const statuses = sentThreadStatuses;
-    expect(statuses).toEqual([
-      {
-        chatId: channelId,
-        threadTs,
-        status: expect.any(String),
-        loadingMessages: ["In progress (1/2): Read request"],
-      },
-      {
-        chatId: channelId,
-        threadTs,
-        status: expect.any(String),
-        loadingMessages: ["In progress (2/2): Write answer"],
-      },
-      {
-        chatId: channelId,
-        threadTs,
-        status: "",
-      },
-    ]);
-    expect(slackStatusLabels).toContain(
-      (statuses[0] as { status: string }).status,
-    );
-    expect(slackStatusLabels).toContain(
-      (statuses[1] as { status: string }).status,
-    );
+    expect(phases()).toEqual(["thinking", "idle"]);
   });
 });