@vellumai/assistant 0.11.1 → 0.11.2-staging.2

package/src/__tests__/provider-error-scenarios.test.ts

@@ -232,6 +232,28 @@ describe("RetryProvider — rate limit backoff", () => {
     }
   });
 
+  test("attributes the credential selected for the failed request", async () => {
+    const inner = makeFailing(
+      new ProviderError("invalid key", "anthropic", 401),
+    );
+    const provider = new RetryProvider(inner, {
+      credentialSource: "byok",
+      connectionName: "anthropic-personal",
+    });
+
+    try {
+      await provider.sendMessage(MESSAGES);
+      expect(true).toBe(false);
+    } catch (err) {
+      expect(err).toBeInstanceOf(ProviderError);
+      const providerError = err as ProviderError;
+      expect(providerError.routeAttribution).toMatchObject({
+        credentialSource: "byok",
+        connectionName: "anthropic-personal",
+      });
+    }
+  });
+
   test("tags final error with retriesExhausted=true after retry loop gives up (JARVIS-513)", async () => {
     // Transient socket flap from Bun's native fetch: wrapped in a
     // ProviderError but still network-retryable via message pattern match.
@@ -333,6 +355,124 @@ describe("RetryProvider — rate limit backoff", () => {
   });
 });
 
+describe("RetryProvider managed credential refresh", () => {
+  test("reloads managed credentials once and keeps the refreshed provider", async () => {
+    sleepSpy.mockClear();
+    const initial = makeFailing(
+      new ProviderError("assistant key expired", "anthropic", 403, {
+        reason: "invalid_credentials",
+      }),
+      "anthropic",
+    );
+    const refreshed = makeFlaky(
+      0,
+      new ProviderError("unused", "anthropic"),
+      "anthropic",
+    );
+    let refreshCalls = 0;
+    const provider = new RetryProvider(initial, {
+      credentialSource: "vellum-managed",
+      connectionName: "vellum",
+      refreshCredentialProvider: async () => {
+        refreshCalls++;
+        return refreshed;
+      },
+    });
+
+    await expect(provider.sendMessage(MESSAGES)).resolves.toMatchObject({
+      model: "test-model",
+    });
+    await expect(provider.sendMessage(MESSAGES)).resolves.toMatchObject({
+      model: "test-model",
+    });
+
+    expect(initial.calls).toBe(1);
+    expect(refreshed.calls).toBe(2);
+    expect(refreshCalls).toBe(1);
+    expect(sleepSpy).not.toHaveBeenCalled();
+  });
+
+  test("surfaces a managed auth failure when refreshed credentials are also rejected", async () => {
+    const initial = makeFailing(
+      new ProviderError("old key rejected", "gemini", 401),
+      "gemini",
+    );
+    const refreshed = makeFailing(
+      new ProviderError("new key rejected", "gemini", 403),
+      "gemini",
+    );
+    let refreshCalls = 0;
+    const provider = new RetryProvider(initial, {
+      credentialSource: "vellum-managed",
+      connectionName: "vellum",
+      refreshCredentialProvider: async () => {
+        refreshCalls++;
+        return refreshed;
+      },
+    });
+
+    try {
+      await provider.sendMessage(MESSAGES);
+      expect.unreachable();
+    } catch (error) {
+      expect(error).toBeInstanceOf(ProviderError);
+      const providerError = error as ProviderError;
+      expect(providerError.message).toBe("new key rejected");
+      expect(providerError.routeAttribution).toMatchObject({
+        credentialSource: "vellum-managed",
+        connectionName: "vellum",
+      });
+    }
+    expect(initial.calls).toBe(1);
+    expect(refreshed.calls).toBe(1);
+    expect(refreshCalls).toBe(1);
+  });
+
+  test("does not reload credentials for a personal provider key rejection", async () => {
+    const initial = makeFailing(
+      new ProviderError("personal key rejected", "anthropic", 401),
+      "anthropic",
+    );
+    let refreshCalls = 0;
+    const provider = new RetryProvider(initial, {
+      credentialSource: "byok",
+      refreshCredentialProvider: async () => {
+        refreshCalls++;
+        return null;
+      },
+    });
+
+    await expect(provider.sendMessage(MESSAGES)).rejects.toThrow(
+      "personal key rejected",
+    );
+    expect(initial.calls).toBe(1);
+    expect(refreshCalls).toBe(0);
+  });
+
+  test("does not reload managed credentials for a model restriction", async () => {
+    const initial = makeFailing(
+      new ProviderError("model unavailable", "anthropic", 403, {
+        reason: "model_restricted",
+      }),
+      "anthropic",
+    );
+    let refreshCalls = 0;
+    const provider = new RetryProvider(initial, {
+      credentialSource: "vellum-managed",
+      refreshCredentialProvider: async () => {
+        refreshCalls++;
+        return null;
+      },
+    });
+
+    await expect(provider.sendMessage(MESSAGES)).rejects.toThrow(
+      "model unavailable",
+    );
+    expect(initial.calls).toBe(1);
+    expect(refreshCalls).toBe(0);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // RetryProvider — ContextOverflowError short-circuit
 // ---------------------------------------------------------------------------

package/src/__tests__/provider-send-message-override-profile.test.ts

@@ -27,6 +27,7 @@ import type {
   ProviderResponse,
   SendMessageOptions,
 } from "../providers/types.js";
+import { ProviderError } from "../util/errors.js";
 import { setConfig } from "./helpers/set-config.js";
 
 const DUMMY_MESSAGES: Message[] = [
@@ -239,6 +240,91 @@ describe("SendMessageOptions.config.overrideProfile", () => {
     expect(response.model).toBe("openai");
   });
 
+  test("CallSiteRoutingProvider attributes provider errors to the actual resolved connection", async () => {
+    setLlmConfig({
+      profiles: {
+        fast: {
+          provider: "openai",
+          provider_connection: "openai-conn",
+          model: "gpt-5.4",
+        },
+      },
+    });
+
+    const providerError = new ProviderError(
+      "OpenAI API error (429): Too many requests",
+      "openai",
+      429,
+      { reason: "rate_limited" },
+    );
+    const defaultProvider = makeThrowingProvider(
+      "anthropic",
+      new Error("default provider should not be called"),
+    );
+    const altProvider = makeThrowingProvider("openai", providerError);
+    altProvider.routeAttribution = {
+      connectionName: "openai-recovered",
+      isManagedRoute: false,
+    };
+    const wrapped = new CallSiteRoutingProvider(
+      defaultProvider,
+      async (connectionName) =>
+        connectionName === "openai-conn" ? altProvider : null,
+      { connectionName: "vellum", isManagedRoute: true },
+    );
+
+    await expect(
+      wrapped.sendMessage(DUMMY_MESSAGES, {
+        config: { callSite: "mainAgent", overrideProfile: "fast" },
+      }),
+    ).rejects.toBe(providerError);
+    expect(providerError.routeAttribution).toEqual({
+      connectionName: "openai-recovered",
+      profileName: "fast",
+      isManagedRoute: false,
+    });
+  });
+
+  test("CallSiteRoutingProvider attributes soft fallback errors to the actual default connection", async () => {
+    setLlmConfig({
+      profiles: {
+        fast: {
+          provider: "openai",
+          provider_connection: "openai-conn",
+          model: "gpt-5.4",
+        },
+      },
+    });
+
+    const providerError = new ProviderError(
+      "Anthropic API error (429): Too many requests",
+      "anthropic",
+      429,
+      { reason: "rate_limited" },
+    );
+    const defaultProvider = makeThrowingProvider("anthropic", providerError);
+    defaultProvider.routeAttribution = {
+      connectionName: "anthropic-default",
+      isManagedRoute: false,
+    };
+    const wrapped = new CallSiteRoutingProvider(
+      defaultProvider,
+      async () => null,
+      defaultProvider.routeAttribution,
+    );
+
+    await expect(
+      wrapped.sendMessage(DUMMY_MESSAGES, {
+        config: { callSite: "mainAgent", overrideProfile: "fast" },
+      }),
+    ).rejects.toBe(providerError);
+    expect(providerError.routeAttribution).toEqual({
+      connectionName: "anthropic-default",
+      profileName: "fast",
+      isManagedRoute: false,
+    });
+  });
+
   test("missing overrideProfile name silently falls through to base resolution", async () => {
     setLlmConfig({
       // The call-site tweak applies last in resolution, so it pins the model
@@ -328,6 +414,15 @@ describe("SendMessageOptions.config.overrideProfile", () => {
   });
 });
 
+function makeThrowingProvider(name: string, error: Error): Provider {
+  return {
+    name,
+    async sendMessage(): Promise<ProviderResponse> {
+      throw error;
+    },
+  };
+}
+
 describe("SendMessageOptions.config.forceOverrideProfile", () => {
   test("CallSiteConfiguredProvider forwards forceOverrideProfile into the send config", async () => {
     let captured: SendMessageOptions | undefined;

package/src/__tests__/run-conversation-turn-persistence.test.ts

@@ -5,6 +5,10 @@ import {
   ensureConversationExists,
   getConversation,
 } from "../persistence/conversation-crud.js";
+import {
+  isEchoSuppressedUserMessage,
+  isReplyPushIneligibleUserMessage,
+} from "../persistence/conversation-types.js";
 import { getDb } from "../persistence/db-connection.js";
 import { initializeDb } from "../persistence/db-init.js";
 
@@ -31,6 +35,9 @@ mock.module("../runtime/sync/resource-sync-events.js", () => ({
 // `ensureConversationExists`. The persistence module is intentionally NOT
 // mocked so the real `ensureConversationExists` runs against the real DB.
 let lastProcessMessageConversationId: string | undefined;
+let lastProcessMessageOptions: Record<string, unknown> | undefined;
+let lastEnqueueOptions: Record<string, unknown> | undefined;
+let conversationIsProcessing = false;
 mock.module("../daemon/conversation-store.js", () => ({
   getOrCreateConversation: async (
     conversationId: string,
@@ -50,14 +57,18 @@ mock.module("../daemon/conversation-store.js", () => ({
     }
     return {
       abortController: undefined,
-      isProcessing: () => false,
-      async processMessage() {
+      isProcessing: () => conversationIsProcessing,
+      async processMessage(processOptions: Record<string, unknown>) {
         // The row must already exist by the time the turn persists its user
         // message — record the id so the FK precondition can be asserted.
         lastProcessMessageConversationId = conversationId;
+        lastProcessMessageOptions = processOptions;
         return "user-message-id";
       },
-      enqueueMessage: () => ({ rejected: false }),
+      enqueueMessage: (enqueueOptions: Record<string, unknown>) => {
+        lastEnqueueOptions = enqueueOptions;
+        return { rejected: false };
+      },
     };
   },
 }));
@@ -82,6 +93,9 @@ describe("runConversationTurn persistence", () => {
     db.run("DELETE FROM conversations");
     listChangedCalls.length = 0;
     lastProcessMessageConversationId = undefined;
+    lastProcessMessageOptions = undefined;
+    lastEnqueueOptions = undefined;
+    conversationIsProcessing = false;
   });
 
   test("persists a conversations row for a freshly-minted conversation", async () => {
@@ -130,3 +144,52 @@ describe("runConversationTurn persistence", () => {
     expect(listChangedCalls).toEqual([]);
   });
 });
+
+// A plugin drives its turn on its own schedule, so the row that opens it is
+// machine-initiated even when the turn runs in an ordinary standard
+// conversation the user also types into. Each case asserts the shared
+// eligibility predicate's verdict on the stamped metadata, so the marker and
+// the gate that reads it cannot drift apart.
+describe("runConversationTurn provenance", () => {
+  beforeEach(() => {
+    const db = getDb();
+    db.run("DELETE FROM messages");
+    db.run("DELETE FROM conversations");
+    lastProcessMessageOptions = undefined;
+    lastEnqueueOptions = undefined;
+    conversationIsProcessing = false;
+  });
+
+  test("stamps the initiating row automated so its reply raises no push", async () => {
+    const existing = createConversation({ title: "standard conversation" });
+
+    await runConversationTurn({
+      conversationId: existing.id,
+      content: [{ type: "text", text: "transcript excerpt" }],
+    });
+
+    const metadata = lastProcessMessageOptions?.metadata as Record<
+      string,
+      unknown
+    >;
+    expect(metadata).toEqual({ automated: true });
+    expect(isReplyPushIneligibleUserMessage(metadata)).toBe(true);
+    // Not an echo-suppression marker: the row still renders in the transcript.
+    expect(isEchoSuppressedUserMessage(metadata)).toBe(false);
+  });
+
+  test("stamps the same marker on a turn queued behind a busy conversation", async () => {
+    const existing = createConversation({ title: "busy conversation" });
+    conversationIsProcessing = true;
+
+    const result = await runConversationTurn({
+      conversationId: existing.id,
+      content: [{ type: "text", text: "transcript excerpt" }],
+    });
+
+    expect(result.queued).toBe(true);
+    const metadata = lastEnqueueOptions?.metadata as Record<string, unknown>;
+    expect(metadata).toEqual({ automated: true });
+    expect(isReplyPushIneligibleUserMessage(metadata)).toBe(true);
+  });
+});

package/src/__tests__/scripted-turn-metadata-persistence.test.ts

@@ -0,0 +1,209 @@
+/**
+ * Verifies how `persistQueuedMessageBody` resolves the `scripted` marker onto
+ * `messages.metadata.scripted`, which `turn-events-store` forwards to
+ * `TurnTelemetryEvent.scripted`.
+ *
+ * `scripted` is what lets activation metrics exclude auto-sent onboarding
+ * turns for EVERY owner, rather than only those who opted into diagnostics
+ * (ANT-10). Its contract is three-state and the states are not
+ * interchangeable:
+ *
+ *   true    - auto-sent on the user's behalf
+ *   false   - a genuine typed user message
+ *   absent  - UNKNOWN
+ *
+ * The absent case is the one worth protecting: downstream falls back to the
+ * legacy trace-text classifier when the flag is missing, but TRUSTS an
+ * explicit `false`. So a spurious `false` is strictly worse than no value:
+ * it re-inflates activation past the point where the fallback can catch it.
+ *
+ * Mirrors the mock harness of `client-os-metadata-persistence.test.ts`:
+ * exercises `persistQueuedMessageBody` directly with a captured `addMessage`.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+const addMessageCalls: Array<{
+  conversationId: string;
+  role: string;
+  content: string;
+  metadata?: Record<string, unknown>;
+}> = [];
+
+mock.module("../persistence/conversation-crud.js", () => ({
+  setConversationProcessingStartedAt: () => {},
+  isConversationProcessing: () => false,
+  addMessage: async (
+    conversationId: string,
+    role: string,
+    content: string,
+    options?: { metadata?: Record<string, unknown> },
+  ) => {
+    addMessageCalls.push({
+      conversationId,
+      role,
+      content,
+      metadata: options?.metadata,
+    });
+    return { id: `persisted-${addMessageCalls.length}` };
+  },
+  getConversation: () => null,
+  provenanceFromTrustContext: () => ({}),
+  setConversationOriginChannelIfUnset: () => {},
+  setConversationOriginInterfaceIfUnset: () => {},
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
+}));
+
+mock.module("../persistence/conversation-disk-view.js", () => ({
+  syncMessageToDisk: () => {},
+  updateMetaFile: () => {},
+}));
+
+mock.module("../persistence/attachments-store.js", () => ({
+  attachmentExists: () => false,
+  linkAttachmentToMessage: () => {},
+  attachInlineAttachmentToMessage: () => {},
+  validateAttachmentUpload: () => ({ ok: true }),
+  AttachmentUploadError: class extends Error {},
+}));
+
+import type {
+  TurnChannelContext,
+  TurnInterfaceContext,
+} from "../channels/types.js";
+import type { MessagingConversationContext } from "../daemon/conversation-messaging.js";
+import { persistQueuedMessageBody } from "../daemon/conversation-messaging.js";
+import type { MessageQueue } from "../daemon/conversation-queue-manager.js";
+
+function createContext(): MessagingConversationContext {
+  const channel: TurnChannelContext = {
+    userMessageChannel: "vellum",
+    assistantMessageChannel: "vellum",
+  };
+  const iface: TurnInterfaceContext = {
+    userMessageInterface: "web",
+    assistantMessageInterface: "web",
+  };
+  const queueStub = {
+    push: () => true,
+    drain: () => [],
+    size: () => 0,
+  } as unknown as MessageQueue;
+  let processing = false;
+  return {
+    conversationId: "conv-scripted-test",
+    messages: [],
+    isProcessing: () => processing,
+    setProcessing: (value: boolean) => {
+      processing = value;
+    },
+    abortController: null,
+    queue: queueStub,
+    clientOs: undefined,
+    getTurnChannelContext: () => channel,
+    getTurnInterfaceContext: () => iface,
+  };
+}
+
+function lastUserMetadata(): Record<string, unknown> {
+  expect(addMessageCalls.length).toBeGreaterThan(0);
+  const metadata = addMessageCalls.at(-1)?.metadata;
+  expect(metadata).toBeDefined();
+  return metadata!;
+}
+
+describe("scripted-turn metadata persistence", () => {
+  beforeEach(() => {
+    addMessageCalls.length = 0;
+  });
+
+  test("stamps the typed option", async () => {
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "[User action on choice surface: Work]",
+      requestId: "req-option-true",
+      scripted: true,
+    });
+
+    expect(lastUserMetadata().scripted).toBe(true);
+  });
+
+  test("carries the flag through the metadata bag for queued sends", async () => {
+    // The queue round-trips `metadata` but not `PersistMessageOptions`, so a
+    // surface action that was enqueued while a turn was in flight can only
+    // deliver its marker this way. If this regresses, queued surface actions
+    // silently become "unknown" while direct ones stay marked.
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "[User action on app: Answer Selected]",
+      requestId: "req-metadata-true",
+      metadata: { scripted: true },
+    });
+
+    expect(lastUserMetadata().scripted).toBe(true);
+  });
+
+  test("treats `automated` as implying scripted", async () => {
+    // Machine-authored by definition, so it is not a turn the user typed.
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "automated skill message",
+      requestId: "req-automated",
+      metadata: { automated: true },
+    });
+
+    expect(lastUserMetadata().scripted).toBe(true);
+  });
+
+  test("lets an explicit option override the automated default", async () => {
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "automated but counts as a real turn",
+      requestId: "req-automated-override",
+      metadata: { automated: true },
+      scripted: false,
+    });
+
+    expect(lastUserMetadata().scripted).toBe(false);
+  });
+
+  test("defaults an ordinary send to false, asserting a typed turn", async () => {
+    // The default is what makes activation MEASURABLE: absent would mean
+    // "unknown", which is strictly worse information than a truthful false.
+    // Safe only because every auto-send path is marked at its source (web
+    // onboarding flows, surface synthetics, `automated`).
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "a message the user actually typed",
+      requestId: "req-ordinary",
+    });
+
+    expect(lastUserMetadata().scripted).toBe(false);
+  });
+
+  test("falls back to the default for a non-boolean `scripted` in the bag", async () => {
+    // Guards against a truthy string ("true", "1") from an untyped caller
+    // being read as a scripted assertion. It must not survive the metadata
+    // spread either: sqlite would store the string verbatim and the store's
+    // narrowing turns anything that isn't 1 into `false`, so a leaked "true"
+    // would invert into "the user typed this".
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "hello",
+      requestId: "req-bogus",
+      metadata: { scripted: "true" },
+    });
+
+    expect(lastUserMetadata().scripted).toBe(false);
+  });
+
+  test("stamps false when the caller explicitly asserts a typed turn", async () => {
+    const ctx = createContext();
+    await persistQueuedMessageBody(ctx, {
+      content: "hello",
+      requestId: "req-explicit-false",
+      scripted: false,
+    });
+
+    expect(lastUserMetadata().scripted).toBe(false);
+  });
+});

package/src/__tests__/skills.test.ts

@@ -798,6 +798,33 @@ describe("always-candidate frontmatter parsing", () => {
     const wf = loadSkillCatalog().find((s) => s.id === "workflows");
     expect(wf?.alwaysCandidate).toBe(true);
   });
+
+  test("the bundled visualize skill is flagged always-candidate", () => {
+    const viz = loadSkillCatalog().find((s) => s.id === "visualize");
+    expect(viz?.alwaysCandidate).toBe(true);
+  });
+
+  test("every always-candidate bundled skill's card fits its budget", async () => {
+    // The card is pinned into the selector's stable prefix every turn, and
+    // `buildSkillContent` hard-truncates at the budget — a skill that overruns
+    // silently loses its last activation hints and its avoid-when list, which
+    // is exactly the guidance the pinning was added to deliver.
+    const { ALWAYS_CANDIDATE_CARD_CHARS, buildSkillContent } =
+      await import("../plugins/defaults/memory/substrate/skill-content.js");
+    const pinned = loadSkillCatalog().filter(
+      (skill) => skill.bundled && skill.alwaysCandidate === true,
+    );
+    expect(pinned.length).toBeGreaterThan(0);
+
+    const overrun = pinned
+      .map((skill) => ({
+        id: skill.id,
+        chars: buildSkillContent(skill, Number.MAX_SAFE_INTEGER).length,
+      }))
+      .filter(({ chars }) => chars > ALWAYS_CANDIDATE_CARD_CHARS)
+      .map(({ id, chars }) => `${id}: ${chars}`);
+    expect(overrun).toEqual([]);
+  });
 });
 
 describe("bundled skill categories", () => {

package/src/__tests__/slack-channels-routes.test.ts

@@ -40,8 +40,6 @@ mock.module("../messaging/providers/slack/client.js", () => ({
     }
     throw new Error(`User not found: ${userId}`);
   },
-  // auth.ts imports SlackApiError from the client; export it from the mock.
-  SlackApiError: class SlackApiError extends Error {},
 }));
 
 // ---------------------------------------------------------------------------

package/src/__tests__/slack-share-routes.test.ts

@@ -51,9 +51,6 @@ mock.module("../messaging/providers/slack/client.js", () => ({
     _text: string,
     _opts?: unknown,
   ) => postMessageResult,
-  // auth.ts imports SlackApiError from the client; export it so the import
-  // graph loads (this file never triggers the fallback that inspects it).
-  SlackApiError: class SlackApiError extends Error {},
 }));
 
 let appStoreResult: unknown = null;

package/src/__tests__/slack-users-routes.test.ts

@@ -35,8 +35,6 @@ mock.module("../messaging/providers/slack/client.js", () => ({
     listUsersCalls.push({ cursor });
     return listUsersPages[listUsersCalls.length - 1];
   },
-  // auth.ts imports SlackApiError from the client; export it from the mock.
-  SlackApiError: class SlackApiError extends Error {},
 }));
 
 // ---------------------------------------------------------------------------

package/src/__tests__/subagent-tools.test.ts

@@ -1691,6 +1691,17 @@ describe("Subagent advisor-role consult", () => {
       expect(captured.current!.config.systemPromptOverride).toContain(
         "PARENT SYSTEM PROMPT",
       );
+      // The situational context pack must never ride display surfaces: the
+      // system prompt stays minimal and `objective` (rendered verbatim by the
+      // subagent detail panel) stays the concise advice request. Only the
+      // non-display `requestText` may carry the pack.
+      expect(captured.current!.config.systemPromptOverride).not.toContain(
+        "<agent_environment>",
+      );
+      expect(captured.current!.config.objective).not.toContain(
+        "<agent_environment>",
+      );
+      expect(captured.current!.config.requestText).toContain("advise me");
     } finally {
       restore();
       mockFindConversation = () => undefined;