@vellumai/assistant 0.8.2 → 0.8.4

package/src/context/window-manager.ts

@@ -51,6 +51,11 @@ export interface ContextWindowResult {
   thresholdTokens: number;
   compactedMessages: number;
   compactedPersistedMessages: number;
+  /**
+   * Number of recent ("tail") messages preserved verbatim alongside the
+   * summary. Omitted on no-op / skipped results — defaults to 0 at render.
+   */
+  preservedTailMessages?: number;
   summaryCalls: number;
   summaryInputTokens: number;
   summaryOutputTokens: number;
@@ -216,6 +221,26 @@ export class ContextWindowManager {
     return getConfig().compaction;
   }
 
+  get maxInputTokens(): number {
+    return this.config.maxInputTokens;
+  }
+
+  /**
+   * Estimate the prompt-token cost of `messages` using the same path as the
+   * auto-compaction pre-check. Clears the system-prompt cache so the next
+   * turn re-resolves it (the system prompt is lazy and may have changed).
+   */
+  estimateInputTokens(messages: Message[]): number {
+    try {
+      return estimatePromptTokens(messages, this.systemPrompt, {
+        providerName: this.estimationProviderName,
+        toolTokenBudget: this.toolTokenBudget,
+      });
+    } finally {
+      this.clearSystemPromptCache();
+    }
+  }
+
   /**
    * Cheap pre-check — estimate the current token count and compare against
    * `compaction.autoThreshold`. Callers pass the estimate back through

package/src/credential-health/credential-health-service.ts

@@ -315,9 +315,11 @@ async function checkManagedProvider(
     const client = await VellumPlatformClient.create();
     if (!client?.platformAssistantId) return results;
 
+    // Query without a status filter so we can distinguish "never
+    // connected" (empty result) from "previously connected but now
+    // inactive" (non-empty result with no ACTIVE entries).
     const params = new URLSearchParams();
     params.set("provider", providerRow.provider);
-    params.set("status", "ACTIVE");
 
     const path = `/v1/assistants/${encodeURIComponent(client.platformAssistantId)}/oauth/connections/?${params.toString()}`;
     const response = await client.fetch(path);
@@ -331,19 +333,30 @@ async function checkManagedProvider(
     }
 
     const body = (await response.json()) as unknown;
-    const connections = (
+    const allConnections = (
       Array.isArray(body)
         ? body
         : ((body as Record<string, unknown>).results ?? [])
-    ) as Array<{ id: string; account_label?: string }>;
+    ) as Array<{ id: string; account_label?: string; status?: string }>;
+
+    if (allConnections.length === 0) {
+      // No connections of any status — the user has never connected this
+      // provider. The suggested-prompts system handles prompting them to
+      // connect; this is not a health issue.
+      return results;
+    }
+
+    const connections = allConnections.filter(
+      (c) => (c.status ?? "ACTIVE").toUpperCase() === "ACTIVE",
+    );
 
     if (connections.length === 0) {
-      // No active managed connections — report as missing so the
-      // heartbeat can notify the user.
+      // Connections exist but none are active — the user previously
+      // connected and the connection was revoked/deactivated.
       results.push({
         connectionId: `managed:${providerRow.provider}`,
         provider: providerRow.provider,
-        accountInfo: null,
+        accountInfo: allConnections[0]?.account_label ?? null,
         status: "missing_token",
         details: `No active managed connection for ${providerRow.provider}. Reconnect on the Vellum platform.`,
         missingScopes: [],
@@ -550,9 +563,20 @@ export async function checkAllCredentials(): Promise<CredentialHealthReport> {
   for (const providerRow of providers) {
     if (!(await isManagedProvider(providerRow))) continue;
 
-    // If the provider is in managed mode and also has BYO connections,
-    // remove the stale BYO results — managed mode takes priority.
-    if (byoProviders.has(providerRow.provider)) {
+    let managedResults: CredentialHealthResult[] = [];
+    try {
+      managedResults = await checkManagedProvider(providerRow);
+    } catch (err) {
+      log.warn(
+        { err, provider: providerRow.provider },
+        "Failed to check managed provider health",
+      );
+    }
+
+    // Only replace BYO results with managed results when the managed
+    // check returned something. If managed returned empty (user never
+    // connected via managed mode), keep any existing BYO results.
+    if (managedResults.length > 0 && byoProviders.has(providerRow.provider)) {
       const beforeLen = results.length;
       const filtered = results.filter(
         (r) => r.provider !== providerRow.provider,
@@ -562,16 +586,7 @@ export async function checkAllCredentials(): Promise<CredentialHealthReport> {
         results.push(...filtered);
       }
     }
-
-    try {
-      const managedResults = await checkManagedProvider(providerRow);
-      results.push(...managedResults);
-    } catch (err) {
-      log.warn(
-        { err, provider: providerRow.provider },
-        "Failed to check managed provider health",
-      );
-    }
+    results.push(...managedResults);
   }
 
   const unhealthy = results.filter((r) => r.status !== "healthy");

package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts

@@ -97,7 +97,6 @@ mock.module("../../memory/auto-analysis-enqueue.js", () => ({
   },
 }));
 
-let memoryRetroEnabled = false;
 const memoryRetroCalls: Array<{
   conversationId: string;
   trigger: string;
@@ -108,7 +107,6 @@ mock.module("../../memory/memory-retrospective-enqueue.js", () => ({
     conversationId: string;
     trigger: string;
   }) => {
-    if (!memoryRetroEnabled) return;
     memoryRetroCalls.push(args);
   },
   // Also export sibling functions other modules import from this file, so
@@ -205,7 +203,6 @@ describe("disposeConversation — auto-analysis enqueue", () => {
     autoAnalyzeCalls.length = 0;
     memoryRetroCalls.length = 0;
     autoAnalyzeEnabled = true;
-    memoryRetroEnabled = false;
     autoAnalysisConversations.clear();
     v2Enabled = false;
   });
@@ -390,13 +387,11 @@ describe("disposeConversation — memory-retrospective lifecycle safety net", ()
     autoAnalyzeCalls.length = 0;
     memoryRetroCalls.length = 0;
     autoAnalyzeEnabled = false;
-    memoryRetroEnabled = false;
     autoAnalysisConversations.clear();
     v2Enabled = false;
   });
 
-  test("guardian conversation + flag on — enqueues memory-retrospective with trigger 'lifecycle'", () => {
-    memoryRetroEnabled = true;
+  test("guardian conversation — enqueues memory-retrospective with trigger 'lifecycle'", () => {
     const ctx = makeDisposeContext({
       conversationId: "conv-retro",
       trustClass: "guardian",
@@ -411,20 +406,7 @@ describe("disposeConversation — memory-retrospective lifecycle safety net", ()
     });
   });
 
-  test("flag off — no memory-retrospective enqueue", () => {
-    memoryRetroEnabled = false;
-    const ctx = makeDisposeContext({
-      conversationId: "conv-retro-off",
-      trustClass: "guardian",
-    });
-
-    disposeConversation(ctx);
-
-    expect(memoryRetroCalls).toHaveLength(0);
-  });
-
-  test("untrusted actor — no memory-retrospective enqueue even when flag is on", () => {
-    memoryRetroEnabled = true;
+  test("untrusted actor — no memory-retrospective enqueue", () => {
     const ctx = makeDisposeContext({
       conversationId: "conv-retro-untrusted",
       trustClass: "unknown",
@@ -443,8 +425,7 @@ describe("disposeConversation — memory-retrospective lifecycle safety net", ()
   // outside the `!isAutoAnalysis` guard, so it fired even for auto-analysis
   // conversations. Mirrors the indexer-time gate in `indexer.ts` and
   // matches the existing graph_extract recursion-guard semantics.
-  test("auto-analysis conversation — does NOT enqueue memory-retrospective even with flag on", () => {
-    memoryRetroEnabled = true;
+  test("auto-analysis conversation — does NOT enqueue memory-retrospective", () => {
     autoAnalysisConversations.add("conv-auto-retro");
     const ctx = makeDisposeContext({
       conversationId: "conv-auto-retro",

package/src/daemon/__tests__/conversation-tool-setup.test.ts

@@ -49,11 +49,8 @@ mock.module("../../runtime/assistant-event-hub.js", () => ({
 
 // Dynamic imports after mock.module calls so the stubs take effect
 // before the modules under test are loaded.
-const {
-  HOST_TOOL_NAMES,
-  HOST_TOOL_TO_CAPABILITY,
-  isToolActiveForContext,
-} = await import("../conversation-tool-setup.js");
+const { HOST_TOOL_NAMES, HOST_TOOL_TO_CAPABILITY, isToolActiveForContext } =
+  await import("../conversation-tool-setup.js");
 type SkillProjectionContext =
   import("../conversation-tool-setup.js").SkillProjectionContext;
 type SkillProjectionCache =
@@ -75,6 +72,66 @@ beforeEach(() => {
   mockClientCountByCapability.clear();
 });
 
+describe("isToolActiveForContext — Slack task_progress UI exception", () => {
+  test("ui_show and ui_update are active for Slack task_progress turns", () => {
+    const ctx = makeCtx({
+      hasNoClient: false,
+      channelCapabilities: {
+        channel: "slack",
+        supportsDynamicUi: false,
+      },
+    });
+
+    expect(isToolActiveForContext("ui_show", ctx)).toBe(true);
+    expect(isToolActiveForContext("ui_update", ctx)).toBe(true);
+  });
+
+  test("ui_dismiss remains hidden for Slack without dynamic UI support", () => {
+    expect(
+      isToolActiveForContext(
+        "ui_dismiss",
+        makeCtx({
+          hasNoClient: false,
+          channelCapabilities: {
+            channel: "slack",
+            supportsDynamicUi: false,
+          },
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  test("Slack task_progress UI tools still require a connected client path", () => {
+    expect(
+      isToolActiveForContext(
+        "ui_show",
+        makeCtx({
+          hasNoClient: true,
+          channelCapabilities: {
+            channel: "slack",
+            supportsDynamicUi: false,
+          },
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  test("other non-dynamic channels still hide UI surface tools", () => {
+    expect(
+      isToolActiveForContext(
+        "ui_show",
+        makeCtx({
+          hasNoClient: false,
+          channelCapabilities: {
+            channel: "telegram",
+            supportsDynamicUi: false,
+          },
+        }),
+      ),
+    ).toBe(false);
+  });
+});
+
 describe("isToolActiveForContext — host tool capability gating", () => {
   // macOS transport: SSE-based interactive approval required.
   test("host_bash is active for macOS with a connected client", () => {
@@ -331,7 +388,10 @@ describe("isToolActiveForContext — cross-client exposure for host_file_*", ()
       expect(
         isToolActiveForContext(
           tool,
-          makeCtx({ hasNoClient: true, transportInterface: "chrome-extension" }),
+          makeCtx({
+            hasNoClient: true,
+            transportInterface: "chrome-extension",
+          }),
         ),
       ).toBe(false);
     });

package/src/daemon/__tests__/native-web-search-metadata.test.ts

@@ -0,0 +1,357 @@
+/**
+ * Tests that the native Anthropic `server_tool_complete` agent event produces
+ * a structured `WebSearchMetadata` payload on the emitted `tool_result` server
+ * message while keeping the back-compat `result: string` byte-identical.
+ *
+ * Mirrors the mocked-dependency collector pattern used in
+ * tool-result-metadata-plumbing.test.ts.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock platform (must precede imports that read it) ─────────────────────────
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+}));
+
+mock.module("../../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  dispatchAgentEvent,
+} from "../conversation-agent-loop-handlers.js";
+import type { ServerMessage } from "../message-protocol.js";
+
+type ToolResultEvent = Extract<ServerMessage, { type: "tool_result" }>;
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function createCollectorDeps(providerName = "anthropic"): {
+  deps: EventHandlerDeps;
+  events: ServerMessage[];
+} {
+  const events: ServerMessage[] = [];
+  const deps = {
+    ctx: {
+      conversationId: "conv-native-meta",
+      provider: { name: providerName },
+      traceEmitter: { emit: () => {} },
+      streamThinking: false,
+      emitActivityState: () => {},
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (msg: ServerMessage) => events.push(msg),
+    reqId: "req-native-meta",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "macos",
+      assistantMessageInterface: "macos",
+    } as EventHandlerDeps["turnInterfaceContext"],
+  } as EventHandlerDeps;
+  return { deps, events };
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("native server_tool_complete metadata", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("builds structured WebSearchMetadata and keeps result text byte-identical", async () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu1";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: { query: "Air Jordan auction" },
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "X",
+          url: "https://www.cnn.com/a",
+        },
+        {
+          type: "web_search_result",
+          title: "Y",
+          url: "https://www.nbcnews.com/b",
+        },
+      ],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent).toBeDefined();
+
+    const meta = toolResultEvent?.activityMetadata?.webSearch;
+    expect(meta).toBeDefined();
+    expect(meta?.query).toBe("Air Jordan auction");
+    expect(meta?.provider).toBe("anthropic-native");
+    expect(meta?.resultCount).toBe(2);
+    expect(meta?.results[0]?.domain).toBe("www.cnn.com");
+    expect(meta?.results[1]?.domain).toBe("www.nbcnews.com");
+    expect(meta?.results[0]?.rank).toBe(1);
+    expect(meta?.results[1]?.rank).toBe(2);
+    expect(meta?.durationMs).toBeGreaterThanOrEqual(0);
+
+    // Favicons are synthesized from the result domain so clients can render
+    // a per-result icon without an extra round-trip.
+    expect(meta?.results[0]?.faviconUrl).toContain("google.com/s2/favicons");
+    expect(meta?.results[1]?.faviconUrl).toContain("google.com/s2/favicons");
+
+    // Back-compat: result text must be byte-identical to the legacy format.
+    expect(toolResultEvent?.result).toBe(
+      "X\nhttps://www.cnn.com/a\n\nY\nhttps://www.nbcnews.com/b",
+    );
+
+    // Per-toolUseId scratch maps must be cleaned up after completion.
+    expect(state.serverToolStartedAt.has(toolUseId)).toBe(false);
+    expect(state.serverToolInputs.has(toolUseId)).toBe(false);
+  });
+
+  test("prefers `resolvedInput.query` over the input captured at server_tool_start", async () => {
+    // Mirrors the live Anthropic flow: server_tool_start fires with `input: {}`
+    // because the SDK streams server-tool input via `input_json_delta`, then
+    // the provider populates `resolvedInput` on `server_tool_complete` from
+    // the accumulated JSON. Without this plumb-through, `meta.query` is empty.
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu_resolved";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: {},
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: false,
+      resolvedInput: { query: "OpenAI Dev Day recap" },
+      content: [
+        {
+          type: "web_search_result",
+          title: "Recap",
+          url: "https://example.com/a",
+        },
+      ],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent?.activityMetadata?.webSearch?.query).toBe(
+      "OpenAI Dev Day recap",
+    );
+  });
+
+  test("fires per-toolUseId for parallel server tool calls with independent queries and timings", async () => {
+    const { deps, events } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_a",
+      input: { query: "alpha" },
+    });
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_b",
+      input: { query: "beta" },
+    });
+
+    // Hold long enough that the two completions can't share a duration.
+    await new Promise((r) => setTimeout(r, 20));
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId: "tu_a",
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "A1",
+          url: "https://a.example.com/1",
+        },
+      ],
+    });
+
+    await new Promise((r) => setTimeout(r, 20));
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId: "tu_b",
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "B1",
+          url: "https://b.example.com/1",
+        },
+      ],
+    });
+
+    const toolResults = events.filter(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResults).toHaveLength(2);
+
+    const byId = new Map(
+      toolResults.map((r) => [r.toolUseId, r] as const),
+    );
+    expect(byId.get("tu_a")?.activityMetadata?.webSearch?.query).toBe("alpha");
+    expect(byId.get("tu_b")?.activityMetadata?.webSearch?.query).toBe("beta");
+
+    const durA = byId.get("tu_a")?.activityMetadata?.webSearch?.durationMs ?? 0;
+    const durB = byId.get("tu_b")?.activityMetadata?.webSearch?.durationMs ?? 0;
+    // Each duration is computed against its own startedAt — tu_b should run
+    // longer because we slept once more between its start and its complete.
+    expect(durB).toBeGreaterThanOrEqual(durA);
+  });
+
+  test("forwards provider error codes instead of generic 'Search failed'", async () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu_err_code";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: { query: "over the limit" },
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: true,
+      errorCode: "max_uses_exceeded",
+      content: [],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent?.activityMetadata?.webSearch?.errorMessage).toBe(
+      "max_uses_exceeded",
+    );
+  });
+
+  test("does NOT emit activityMetadata for non-Anthropic providers", async () => {
+    // OpenAI's responses provider shares `server_tool_start`/`server_tool_complete`
+    // for `web_search_call`, but its results live inside the assistant text
+    // stream — emitting "anthropic-native" metadata for an OpenAI search would
+    // mis-label the provider and ship an empty `results` array.
+    const { deps, events } = createCollectorDeps("openai");
+    const toolUseId = "tu_openai";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: {},
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: false,
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent).toBeDefined();
+    expect(toolResultEvent?.activityMetadata).toBeUndefined();
+    // The back-compat `result: string` channel still fires (empty for OpenAI
+    // since the results are woven into the text stream, not structured).
+    expect(toolResultEvent?.result).toBe("");
+  });
+
+  test("sets errorMessage when the search errored", async () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu_err";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: { query: "broken" },
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: true,
+      content: [],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    const meta = toolResultEvent?.activityMetadata?.webSearch;
+    expect(meta?.errorMessage).toBe("Search failed");
+    expect(meta?.resultCount).toBe(0);
+    expect(meta?.results).toEqual([]);
+    expect(toolResultEvent?.isError).toBe(true);
+  });
+});