@vellumai/assistant 0.11.1 → 0.11.2-staging.2

package/src/__tests__/conversation-error.test.ts

@@ -170,6 +170,112 @@ describe("classifyConversationError", () => {
       expect(result.userMessage).toContain("AI provider");
       expect(result.errorCategory).toBe("rate_limit");
     });
+
+    it("uses the ChatGPT OAuth route instead of the OpenAI registry default", () => {
+      providerRoutingSources.openai = "managed-proxy";
+      const err = new ProviderError(
+        "OpenAI API error (429): Too many requests",
+        "openai",
+        429,
+        { reason: "rate_limited" },
+      );
+
+      const result = classifyConversationError(err, {
+        ...baseCtx,
+        connectionName: "chatgpt-subscription",
+        isManagedRoute: false,
+      });
+
+      expect(result.code).toBe("PROVIDER_RATE_LIMIT");
+      expect(result.userMessage).toContain("AI provider");
+      expect(result.errorCategory).toBe("rate_limit");
+    });
+
+    it("prefers the failed call's direct route over stale managed turn attribution", () => {
+      providerRoutingSources.openai = "managed-proxy";
+      const err = new ProviderError(
+        "OpenAI API error (429): Too many requests",
+        "openai",
+        429,
+        { reason: "rate_limited" },
+      );
+      err.attachRouteAttribution({
+        connectionName: "chatgpt-subscription",
+        profileName: "chatgpt",
+        isManagedRoute: false,
+      });
+
+      const result = classifyConversationError(err, {
+        ...baseCtx,
+        connectionName: "vellum",
+        profileName: "managed",
+        isManagedRoute: true,
+      });
+
+      expect(result.code).toBe("PROVIDER_RATE_LIMIT");
+      expect(result.userMessage).toContain("AI provider");
+      expect(result.errorCategory).toBe("rate_limit");
+    });
+
+    it("prefers the failed call's managed fallback over stale direct turn attribution", () => {
+      const err = new ProviderError(
+        "Anthropic API error (429): Too many requests",
+        "anthropic",
+        429,
+        { reason: "rate_limited" },
+      );
+      err.attachRouteAttribution({
+        profileName: "direct",
+        isManagedRoute: true,
+      });
+
+      const result = classifyConversationError(err, {
+        ...baseCtx,
+        connectionName: "anthropic-key",
+        profileName: "direct",
+        isManagedRoute: false,
+      });
+
+      expect(result.code).toBe("MANAGED_USAGE_LIMIT");
+      expect(result.userMessage).toContain("Vellum managed inference");
+      expect(result.errorCategory).toBe("managed_usage_limit");
+    });
+
+    it("does not apply a managed LLM route to a plain rate-limit error", () => {
+      const result = classifyConversationError(
+        new Error("429 Too Many Requests from a tool API"),
+        {
+          ...baseCtx,
+          isManagedRoute: true,
+        },
+      );
+
+      expect(result.code).toBe("PROVIDER_RATE_LIMIT");
+      expect(result.errorCategory).toBe("rate_limit");
+    });
+
+    it("falls back to context for fields the failed call's route omits", () => {
+      const err = new ProviderError("Unauthorized", "anthropic", 401);
+      err.attachRouteAttribution({ profileName: "direct" });
+
+      const result = classifyConversationError(err, {
+        ...baseCtx,
+        connectionName: "anthropic-key",
+      });
+
+      expect(result.connectionName).toBe("anthropic-key");
+      expect(result.profileName).toBe("direct");
+    });
+
+    it("still recognizes a rewrapped Vellum quota body", () => {
+      const result = classifyConversationError(
+        new Error('429 {"code":"daily_quota_exceeded"}'),
+        baseCtx,
+      );
+
+      expect(result.code).toBe("MANAGED_USAGE_LIMIT");
+      expect(result.errorCategory).toBe("managed_usage_limit");
+    });
   });
 
   describe("provider overloaded errors", () => {
@@ -671,6 +777,9 @@ describe("classifyConversationError", () => {
       expect(result.code).toBe("PROVIDER_INVALID_KEY");
       expect(result.retryable).toBe(false);
       expect(result.errorCategory).toBe("provider_invalid_key");
+      expect(result.userMessage).toBe(
+        "Your personal Anthropic API key was rejected by Anthropic. Update that key in Settings → Models & Services.",
+      );
     });
 
     it("classifies managed-proxy auth failures as managed credential refresh failures", () => {
@@ -685,12 +794,30 @@ describe("classifyConversationError", () => {
 
       expect(result.code).toBe("MANAGED_KEY_INVALID");
       expect(result.userMessage).toBe(
-        "Couldn't refresh assistant credentials.",
+        "Vellum's managed inference credential was rejected. This isn't a personal provider API key — Vellum provisions this one, so there's nothing to update in Settings.",
       );
       expect(result.retryable).toBe(false);
       expect(result.errorCategory).toBe("managed_key_invalid");
     });
 
+    it("keeps credential-shaped 4xx failures on the request's managed route", () => {
+      providerRoutingSources.anthropic = "user-key";
+      const err = new ProviderError(
+        "Authentication error: invalid API key",
+        "anthropic",
+        400,
+      );
+      err.attachRouteAttribution({ credentialSource: "vellum-managed" });
+
+      const result = classifyConversationError(err, baseCtx);
+
+      expect(result.code).toBe("MANAGED_KEY_INVALID");
+      expect(result.errorCategory).toBe("managed_key_invalid");
+      expect(result.userMessage).toBe(
+        "Vellum's managed inference credential was rejected. This isn't a personal provider API key — Vellum provisions this one, so there's nothing to update in Settings.",
+      );
+    });
+
     it("classifies ProviderError 401 with 'invalid x-api-key' message as PROVIDER_INVALID_KEY", () => {
       // Regex-match branch — Anthropic's standard 401 wording.
       const err = new ProviderError(
@@ -708,6 +835,8 @@ describe("classifyConversationError", () => {
       const result = classifyConversationError(err, baseCtx);
       expect(result.code).toBe("PROVIDER_INVALID_KEY");
       expect(result.errorCategory).toBe("provider_invalid_key");
+      expect(result.userMessage).toContain("personal OpenAI API key");
+      expect(result.userMessage).toContain("rejected by OpenAI");
     });
 
     it("includes connection/profile attribution in PROVIDER_INVALID_KEY when provided", () => {
@@ -720,7 +849,84 @@ describe("classifyConversationError", () => {
       expect(result.code).toBe("PROVIDER_INVALID_KEY");
       expect(result.connectionName).toBe("my-anthropic");
       expect(result.profileName).toBe("personal");
-      expect(result.userMessage).toContain("personal");
+      expect(result.userMessage).toBe(
+        'Your personal Anthropic API key for profile "personal" (connection "my-anthropic") was rejected by Anthropic. Update that key in Settings → Models & Services.',
+      );
+    });
+
+    it("uses the request's BYOK attribution when the same provider also has a managed route", () => {
+      providerRoutingSources.anthropic = "managed-proxy";
+      const err = new ProviderError("Unauthorized", "anthropic", 401);
+      err.attachRouteAttribution({
+        credentialSource: "byok",
+        connectionName: "anthropic-personal",
+      });
+
+      const result = classifyConversationError(err, baseCtx);
+
+      expect(result.code).toBe("PROVIDER_INVALID_KEY");
+      expect(result.connectionName).toBe("anthropic-personal");
+      expect(result.userMessage).toBe(
+        'Your personal Anthropic API key for connection "anthropic-personal" was rejected by Anthropic. Update that key in Settings → Models & Services.',
+      );
+    });
+
+    it("uses the request's managed attribution when the same provider also has a BYOK route", () => {
+      providerRoutingSources.anthropic = "user-key";
+      const err = new ProviderError("Unauthorized", "anthropic", 401);
+      err.attachRouteAttribution({
+        credentialSource: "vellum-managed",
+        connectionName: "vellum",
+      });
+
+      const result = classifyConversationError(err, baseCtx);
+
+      expect(result.code).toBe("MANAGED_KEY_INVALID");
+      expect(result.userMessage).toBe(
+        "Vellum's managed inference credential was rejected. This isn't a personal provider API key — Vellum provisions this one, so there's nothing to update in Settings.",
+      );
+    });
+
+    it("uses subscription-specific recovery when the same provider also has a managed route", () => {
+      providerRoutingSources.openai = "managed-proxy";
+      const err = new ProviderError("Unauthorized", "openai", 401, {
+        reason: "invalid_credentials",
+      });
+      err.attachRouteAttribution({
+        credentialSource: "oauth-subscription",
+        connectionName: "chatgpt-subscription",
+      });
+
+      const result = classifyConversationError(err, baseCtx);
+
+      expect(result.code).toBe("PROVIDER_API");
+      expect(result.errorCategory).toBe("provider_subscription_auth");
+      expect(result.retryable).toBe(false);
+      expect(result.connectionName).toBe("chatgpt-subscription");
+      expect(result.userMessage).toBe(
+        'Your OpenAI subscription login for connection "chatgpt-subscription" was rejected by OpenAI. Reconnect that account in Settings → Models & Services.',
+      );
+    });
+
+    it("explains when a no-auth endpoint rejects an unauthenticated request", () => {
+      providerRoutingSources["openai-compatible"] = "managed-proxy";
+      const err = new ProviderError("Unauthorized", "openai-compatible", 401, {
+        reason: "invalid_credentials",
+      });
+      err.attachRouteAttribution({
+        credentialSource: "no-auth",
+        connectionName: "local-model-server",
+      });
+
+      const result = classifyConversationError(err, baseCtx);
+
+      expect(result.code).toBe("PROVIDER_API");
+      expect(result.errorCategory).toBe("provider_endpoint_auth_required");
+      expect(result.retryable).toBe(false);
+      expect(result.connectionName).toBe("local-model-server");
+      expect(result.userMessage).toBe(
+        'The OpenAI-compatible endpoint for connection "local-model-server" requires authentication, but that connection is configured without credentials. Configure authentication for that endpoint in Settings → Models & Services.',
+      );
     });
 
     it("classifies direct ProviderError with 402 as provider_billing (non-retryable)", () => {
@@ -788,8 +994,9 @@ describe("classifyConversationError", () => {
       expect(result.code).toBe("PROVIDER_BILLING");
       expect(result.errorCategory).toBe("credits_exhausted");
       expect(result.retryable).toBe(false);
-      expect(result.userMessage).toContain("Add funds");
-      expect(result.userMessage).toContain("assistant");
+      expect(result.userMessage).toBe(
+        "You're out of credits. Add credits in Settings → Billing to continue.",
+      );
     });
 
     it("classifies direct Anthropic, OpenAI, and OpenRouter 402 responses as provider_billing", () => {

package/src/__tests__/conversation-options-turn-scoped-transport.test.ts

@@ -0,0 +1,64 @@
+/**
+ * Tests for `withoutTurnScopedTransport`, the guard that keeps per-message view
+ * state out of the durable `conversationOptions` map.
+ *
+ * `getOrCreateConversation` re-applies the stored transport whenever it rebuilds
+ * an evicted or stale conversation. Anything describing what the client had on
+ * screen for one message must be stripped before it is stored, or a rebuild
+ * hours later (a scheduled wake, a background follow-up) would resurrect an app
+ * the user has since closed and tell the model it is on screen.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import { withoutTurnScopedTransport } from "../daemon/conversation-store.js";
+import type { ConversationCreateOptions } from "../daemon/handlers/shared.js";
+
+describe("withoutTurnScopedTransport", () => {
+  test("drops the visible app while keeping durable transport fields", () => {
+    const options: ConversationCreateOptions = {
+      transport: {
+        channelId: "vellum",
+        interfaceId: "web",
+        clientOs: "macos",
+        clientTimezone: "America/Denver",
+        visibleAppId: "app-on-screen",
+      },
+    };
+
+    const stored = withoutTurnScopedTransport(options);
+
+    expect(stored.transport?.visibleAppId).toBeUndefined();
+    expect(stored.transport?.clientOs).toBe("macos");
+    expect(stored.transport?.clientTimezone).toBe("America/Denver");
+    expect(stored.transport?.interfaceId).toBe("web");
+  });
+
+  test("does not mutate the caller's options, which still drive this turn", () => {
+    const options: ConversationCreateOptions = {
+      transport: {
+        channelId: "vellum",
+        interfaceId: "web",
+        visibleAppId: "app-on-screen",
+      },
+    };
+
+    withoutTurnScopedTransport(options);
+
+    expect(options.transport?.visibleAppId).toBe("app-on-screen");
+  });
+
+  test("passes options through untouched when there is no app in view", () => {
+    const options: ConversationCreateOptions = {
+      transport: { channelId: "vellum", interfaceId: "web", clientOs: "ios" },
+    };
+
+    expect(withoutTurnScopedTransport(options)).toBe(options);
+  });
+
+  test("passes options through untouched when there is no transport", () => {
+    const options: ConversationCreateOptions = {};
+
+    expect(withoutTurnScopedTransport(options)).toBe(options);
+  });
+});

package/src/__tests__/conversation-queue.test.ts

@@ -138,6 +138,12 @@ mock.module("../telemetry/turn-outcome.js", () => ({
   stampTurnOutcome: mockStampTurnOutcome,
 }));
 
+const emitAssistantReplyNotificationMock = mock(async () => {});
+
+mock.module("../notifications/assistant-reply-producer.js", () => ({
+  emitAssistantReplyNotification: emitAssistantReplyNotificationMock,
+}));
+
 let linkAttachmentShouldThrow = false;
 let mockAttachmentIdCounter = 0;
 
@@ -669,6 +675,52 @@ describe("Conversation message queue", () => {
     expect(pendingRuns.length).toBe(3);
   });
 
+  test("[experimental] queued passthrough siblings viewing different apps do NOT batch", async () => {
+    // A batched turn applies only the head's `visibleAppId`, which drives the
+    // `visible_app:` context line. Coalescing messages sent while different
+    // apps were on screen would point "the app" at the head's app for both.
+    const conversation = makeConversation();
+    await conversation.loadFromDb();
+
+    const p1 = conversation.processMessage({
+      content: "msg-1",
+      attachments: [],
+      onEvent: () => {},
+      requestId: "req-1",
+    });
+    await waitForPendingRun(1);
+
+    conversation.enqueueMessage({
+      content: "msg-2",
+      onEvent: () => {},
+      requestId: "req-2",
+      transport: {
+        channelId: "vellum",
+        interfaceId: "web",
+        visibleAppId: "app-a",
+      },
+    });
+    conversation.enqueueMessage({
+      content: "msg-3",
+      onEvent: () => {},
+      requestId: "req-3",
+      transport: {
+        channelId: "vellum",
+        interfaceId: "web",
+        visibleAppId: "app-b",
+      },
+    });
+    expect(conversation.getQueueDepth()).toBe(2);
+
+    await resolveRun(0);
+    await p1;
+    await waitForPendingRun(2);
+    await resolveRun(1);
+    await waitForPendingRun(3);
+
+    expect(pendingRuns.length).toBe(3);
+  });
+
   test("message_queued and message_dequeued events are emitted", async () => {
     const conversation = makeConversation();
     await conversation.loadFromDb();
@@ -1793,6 +1845,80 @@ describe("Batched drain correctness fixes", () => {
     await new Promise((r) => setTimeout(r, 10));
   });
 
+  // The batch's shared reply answers the person's prompt; the trailing row
+  // only rode along behind it. A channel send gets its reply delivered back to
+  // Slack and a hidden marker is nobody's prompt, so pointing the producer at
+  // either would suppress a push the user is waiting on.
+  // expectEcho pins the deliberate asymmetry between the two gates: the
+  // echo broadcast uses the narrower echo-suppression predicate, so a
+  // channel send is push-ineligible yet must still echo to passive devices.
+  const TRAILING_INELIGIBLE_ROW_CASES: Array<{
+    name: string;
+    metadata: Record<string, unknown>;
+    expectEcho: boolean;
+  }> = [
+    { name: "hidden marker", metadata: { hidden: true }, expectEcho: false },
+    {
+      name: "channel send",
+      metadata: { userMessageChannel: "slack" },
+      expectEcho: true,
+    },
+  ];
+
+  for (const { name, metadata, expectEcho } of TRAILING_INELIGIBLE_ROW_CASES) {
+    test(`drainBatch notifies about the genuine prompt, not a trailing ${name}`, async () => {
+      emitAssistantReplyNotificationMock.mockClear();
+      const conversation = makeConversation();
+      await conversation.loadFromDb();
+
+      const p1 = conversation.processMessage({
+        content: "msg-1",
+        attachments: [],
+        requestId: "req-1",
+      });
+      await waitForPendingRun(1);
+
+      conversation.enqueueMessage({
+        content: "batch-prompt-genuine",
+        requestId: "req-prompt",
+      });
+      const trailingEvents: AssistantEvent[] = [];
+      conversation.enqueueMessage({
+        content: "batch-prompt-trailing",
+        requestId: "req-trailing",
+        metadata,
+        onEvent: (e) => {
+          trailingEvents.push(e);
+        },
+      });
+
+      await resolveRun(0);
+      await p1;
+      await waitForPendingRun(2);
+      await resolveRun(1);
+      // Two turns finish here: msg-1's own run, then the batch's shared run.
+      await waitForCondition(
+        () => emitAssistantReplyNotificationMock.mock.calls.length >= 2,
+      );
+
+      const genuineRow = capturedAddMessages.find((m) =>
+        m.content.includes("batch-prompt-genuine"),
+      );
+      expect(genuineRow?.id).toBeDefined();
+
+      const notifyCalls = emitAssistantReplyNotificationMock.mock
+        .calls as unknown as Array<[{ userMessageId: string | undefined }]>;
+      expect(notifyCalls.at(-1)?.[0].userMessageId).toBe(genuineRow!.id);
+
+      const trailingEchoes = trailingEvents.filter(
+        (e) => e.type === "user_message_echo",
+      );
+      expect(trailingEchoes.length).toBe(expectEcho ? 1 : 0);
+
+      await new Promise((r) => setTimeout(r, 10));
+    });
+  }
+
   // Defensive recovery path: buildPassthroughBatch is designed to make
   // the invariant throw unreachable in practice, so neither the head
   // branch (re-dispatch batch.slice(1) to drainBatch/drainSingleMessage/

package/src/__tests__/conversation-retry-route.test.ts

@@ -17,6 +17,15 @@ import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 import { z } from "zod";
 
+// The route's hidden-prompt branch is exactly what these pure predicates
+// decide, so the crud mock below hands back the real ones rather than a
+// restatement that can drift. Imported from the `conversation-types` leaf so
+// naming them here pulls in no DB graph.
+import {
+  isBackgroundEventMetadata,
+  isEchoSuppressedUserMessage,
+} from "../persistence/conversation-types.js";
+
 mock.module("../config/env.js", () => ({
   isHttpAuthDisabled: () => true,
   hasUngatedHttpAuthDisabled: () => false,
@@ -24,13 +33,6 @@ mock.module("../config/env.js", () => ({
 
 mock.module("../daemon/conversation-process.js", () => ({
   formatSummarizeUpToResult: () => "",
-  isEchoSuppressedUserMessage: (
-    metadata: Record<string, unknown> | undefined,
-  ) =>
-    metadata?.hidden === true ||
-    typeof metadata?.backgroundEventSource === "string",
-  isBackgroundEventMetadata: (metadata: Record<string, unknown> | undefined) =>
-    typeof metadata?.backgroundEventSource === "string",
 }));
 
 mock.module("../daemon/handlers/conversations.js", () => ({
@@ -77,6 +79,8 @@ mock.module("../persistence/conversation-crud.js", () => ({
   extractImageSourcePaths: () => undefined,
   forkConversation: () => ({ id: "forked" }),
   getConversation: getConversationMock,
+  isBackgroundEventMetadata,
+  isEchoSuppressedUserMessage,
   provenanceFromTrustContext: () => ({ provenanceTrustClass: "unknown" }),
   setConversationSurfaced: () => null,
   unarchiveConversation: () => true,
@@ -316,6 +320,41 @@ describe("POST /v1/conversations/:id/retry", () => {
     expect(typeof (options as { onEvent?: unknown }).onEvent).toBe("function");
   });
 
+  // The re-run carries none of the anchor's original delivery orchestration,
+  // so a channel or voice anchor's regenerated reply reaches SSE subscribers
+  // alone and the reply push must not read the anchor's origin markers.
+  const OFF_APP_ANCHOR_CASES: Array<{ name: string; metadata: unknown }> = [
+    { name: "Slack", metadata: { userMessageChannel: "slack" } },
+    {
+      name: "voice-session",
+      metadata: { voiceSessionTurn: true, userMessageChannel: "phone" },
+    },
+    { name: "plain in-app", metadata: { userMessageChannel: "vellum" } },
+  ];
+
+  for (const { name, metadata } of OFF_APP_ANCHOR_CASES) {
+    test(`marks the re-run of a ${name} anchor as app-only delivery`, async () => {
+      discardResult = {
+        anchor: anchorRow({ metadata: JSON.stringify(metadata) }),
+        deletedMessageIds: ["assistant-msg-1"],
+      };
+      const ctx = makeConversation();
+      activeConversation = ctx.conversation;
+
+      const res = await callHandler(
+        retryHandler,
+        makeRequest(),
+        { id: "conv-retry-test" },
+        202,
+      );
+      expect(res.status).toBe(202);
+      await settle();
+
+      const [, , options] = ctx.runAgentLoop.mock.calls[0];
+      expect(options).toMatchObject({ replyDeliveredInAppOnly: true });
+    });
+  }
+
   test("legacy background-event anchor (no recorded mode) re-runs hidden AND non-interactive", async () => {
     // Anchors persisted before `backgroundEventInteractive` existed carry only
     // `backgroundEventSource`. Fall back to non-interactive so a retried

package/src/__tests__/conversation-slash.test.ts

@@ -51,7 +51,8 @@ describe("/model list", () => {
     const message = ((await resolveSlash("/model")) as { message: string })
       .message;
     expect(message).toContain("Best results with the most capable model");
-    expect(message).toContain("Fastest responses at lower cost");
+    expect(message).toContain("Cheapest responses, for high-volume work");
+    expect(message).toContain("Fastest responses, with reasoning turned off");
     expect(message).not.toContain("(DeepSeek V4 Flash)");
   });
 
@@ -63,7 +64,8 @@ describe("/model list", () => {
     expect(message).toContain(
       "High-quality results with the most capable model",
     );
-    expect(message).toContain("Fastest responses at lower cost");
+    expect(message).toContain("Cheapest responses, for high-volume work");
+    expect(message).toContain("Fastest responses, with reasoning turned off");
     expect(message).not.toContain("(DeepSeek V4 Flash)");
   });
 });

package/src/__tests__/conversation-summarize-route.test.ts

@@ -257,7 +257,13 @@ describe("POST /v1/conversations/summarize", () => {
 
     await settle();
 
-    expect(ctx.summarizeUpToMessage).toHaveBeenCalledWith("msg-42");
+    // The second arg is the sink the context-window usage push rides. This
+    // route's conversation may hold the store's no-op sender, so it hands in
+    // the broadcast path its result card goes out on.
+    expect(ctx.summarizeUpToMessage).toHaveBeenCalledWith(
+      "msg-42",
+      expect.any(Function),
+    );
     expect(ctx.emitActivityState).toHaveBeenCalledWith(
       "thinking",
       "context_compacting",