@vellumai/assistant 0.8.2 → 0.8.4

package/src/daemon/__tests__/web-search-status-text.test.ts

@@ -0,0 +1,287 @@
+/**
+ * Tests that the in-flight activity status text emitted for `web_search`
+ * and `web_fetch` tool starts surfaces per-call detail (the query / domain)
+ * instead of a generic placeholder. Covers both the Anthropic native
+ * `server_tool_start` path and the non-native `tool_use` path.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock platform (must precede imports that read it) ─────────────────────────
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+}));
+
+mock.module("../../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  dispatchAgentEvent,
+  formatFetchStatusText,
+  formatSearchStatusText,
+} from "../conversation-agent-loop-handlers.js";
+import type { ServerMessage } from "../message-protocol.js";
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+interface ActivityStateCapture {
+  phase: string;
+  reason: string;
+  scope: string;
+  reqId: string;
+  statusText?: string;
+}
+
+function createCollectorDeps(): {
+  deps: EventHandlerDeps;
+  activityStates: ActivityStateCapture[];
+} {
+  const activityStates: ActivityStateCapture[] = [];
+  const deps = {
+    ctx: {
+      conversationId: "conv-status-text",
+      provider: { name: "anthropic" },
+      traceEmitter: { emit: () => {} },
+      streamThinking: false,
+      emitActivityState: (
+        phase: string,
+        reason: string,
+        scope: string,
+        reqId: string,
+        statusText?: string,
+      ) => {
+        activityStates.push({ phase, reason, scope, reqId, statusText });
+      },
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (_msg: ServerMessage) => {},
+    reqId: "req-status-text",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "macos",
+      assistantMessageInterface: "macos",
+    } as EventHandlerDeps["turnInterfaceContext"],
+  } as EventHandlerDeps;
+  return { deps, activityStates };
+}
+
+// ── Pure helper tests ─────────────────────────────────────────────────────────
+
+describe("formatSearchStatusText", () => {
+  test("surfaces the query in quotes", () => {
+    expect(formatSearchStatusText("web_search", "foo")).toBe(
+      'Searching "foo"',
+    );
+  });
+
+  test("truncates queries longer than 60 chars with an ellipsis", () => {
+    const longQuery = "a".repeat(120);
+    const result = formatSearchStatusText("web_search", longQuery);
+    // 57 chars + "..." = 60 char body, wrapped in 'Searching "..."'
+    expect(result).toBe(`Searching "${"a".repeat(57)}..."`);
+  });
+
+  test("preserves 60-char queries as-is (no truncation at the boundary)", () => {
+    const sixty = "a".repeat(60);
+    expect(formatSearchStatusText("web_search", sixty)).toBe(
+      `Searching "${sixty}"`,
+    );
+  });
+
+  test("falls back to the generic phrasing when the query is empty", () => {
+    expect(formatSearchStatusText("web_search", "")).toBe("Searching the web");
+    expect(formatSearchStatusText("web_search", "   ")).toBe(
+      "Searching the web",
+    );
+  });
+
+  test("emits 'Running <tool>' for non-web_search tools", () => {
+    expect(formatSearchStatusText("other_tool", "x")).toBe(
+      "Running other_tool",
+    );
+  });
+});
+
+describe("formatFetchStatusText", () => {
+  test("returns the domain when given a parseable URL", () => {
+    expect(formatFetchStatusText("https://example.com/path")).toBe(
+      "Reading example.com",
+    );
+  });
+
+  test("lowercases the host", () => {
+    expect(formatFetchStatusText("https://EXAMPLE.COM/x")).toBe(
+      "Reading example.com",
+    );
+  });
+
+  test("falls back when the URL is unparseable or missing", () => {
+    expect(formatFetchStatusText("not a url")).toBe("Reading a page");
+    expect(formatFetchStatusText(undefined)).toBe("Reading a page");
+    expect(formatFetchStatusText(42)).toBe("Reading a page");
+  });
+});
+
+// ── Dispatch-path tests ───────────────────────────────────────────────────────
+
+describe("server_tool_start status text", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("native web_search emits 'Searching \"<query>\"'", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_native_q",
+      input: { query: "foo" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe('Searching "foo"');
+  });
+
+  test("native web_search with long query truncates", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+    const longQuery = "a".repeat(120);
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_native_long",
+      input: { query: longQuery },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe(`Searching "${"a".repeat(57)}..."`);
+  });
+
+  test("native web_search with missing query falls back to the generic phrasing", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_native_empty",
+      input: {},
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Searching the web");
+  });
+});
+
+describe("tool_use status text (non-native)", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("web_search emits 'Searching \"<query>\"'", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_ws",
+      name: "web_search",
+      input: { query: "bar" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe('Searching "bar"');
+  });
+
+  test("web_fetch emits 'Reading <domain>'", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_wf",
+      name: "web_fetch",
+      input: { url: "https://www.nytimes.com/article" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Reading www.nytimes.com");
+  });
+
+  test("web_fetch with malformed url falls back to a generic phrase", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_wf_bad",
+      name: "web_fetch",
+      input: { url: "not-a-url" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Reading a page");
+  });
+
+  test("unrelated tools keep the existing 'Running <friendly>' fallback", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_bash",
+      name: "bash",
+      input: { command: "ls" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Running command");
+  });
+});

package/src/daemon/conversation-agent-loop-handlers.ts

@@ -13,6 +13,7 @@ import type {
   TurnChannelContext,
   TurnInterfaceContext,
 } from "../channels/types.js";
+import { getConfig } from "../config/loader.js";
 import { recordEstimate } from "../context/estimator-calibration.js";
 import { getCalibrationProviderKey } from "../context/token-estimator.js";
 import {
@@ -23,12 +24,15 @@ import {
 } from "../memory/conversation-crud.js";
 import {
   backfillMessageIdOnLogs,
+  buildProviderErrorResponsePayload,
   recordRequestLog,
+  setAgentLoopExitReasonOnLatestLog,
 } from "../memory/llm-request-log-store.js";
 import { backfillMemoryRecallLogMessageId } from "../memory/memory-recall-log-store.js";
 import { backfillMemoryV2ActivationMessageId } from "../memory/memory-v2-activation-log-store.js";
 import { getThreadTs } from "../memory/slack-thread-store.js";
 import {
+  formatSlackTimezoneLabel,
   type SlackMessageMetadata,
   writeSlackMetadata,
 } from "../messaging/providers/slack/message-metadata.js";
@@ -44,7 +48,9 @@ import type {
 import type { ContentBlock, ImageContent } from "../providers/types.js";
 import { isContextOverflowError } from "../providers/types.js";
 import { redactSecrets } from "../security/secret-scanner.js";
+import { extractDomain } from "../tools/network/domain-normalize.js";
 import { ProviderError } from "../util/errors.js";
+import { faviconUrlForDomain } from "../util/favicon.js";
 import { getLogger } from "../util/logger.js";
 import type { DirectiveRequest } from "./assistant-attachments.js";
 import {
@@ -58,7 +64,12 @@ import {
   isContextTooLarge,
 } from "./conversation-error.js";
 import { isProviderOrderingError } from "./conversation-slash.js";
+import { resolveTurnTimezoneContext } from "./date-context.js";
 import type { ServerMessage } from "./message-protocol.js";
+import type {
+  WebSearchMetadata,
+  WebSearchResultItem,
+} from "./message-types/web-activity.js";
 
 const log = getLogger("agent-loop-handlers");
 
@@ -197,6 +208,10 @@ export interface EventHandlerState {
   currentTurnToolUseIds: string[];
   /** Wall-clock time (ms since epoch) when the agent loop turn started, used as the display timestamp for assistant messages. */
   turnStartedAt: number;
+  /** Wall-clock start time of native server tool calls, keyed by tool_use_id. */
+  readonly serverToolStartedAt: Map<string, number>;
+  /** Original input from server_tool_start, keyed by tool_use_id, so the complete handler can read the query. */
+  readonly serverToolInputs: Map<string, Record<string, unknown>>;
 }
 
 /** Immutable context shared across event handlers within a single agent loop run. */
@@ -255,6 +270,8 @@ export function createEventHandlerState(): EventHandlerState {
     toolRiskOutcomes: new Map(),
     currentTurnToolUseIds: [],
     turnStartedAt: Date.now(),
+    serverToolStartedAt: new Map(),
+    serverToolInputs: new Map(),
   };
 }
 
@@ -322,6 +339,68 @@ function friendlyToolName(name: string): string {
   return TOOL_FRIENDLY_NAMES[name] ?? name.replace(/_/g, " ");
 }
 
+/**
+ * Status text shown to the client while a web search is in flight.
+ * Surfaces the actual query so the loading state is meaningful instead of
+ * a generic "Searching the web". Used for both Anthropic native server-tool
+ * starts and non-native `web_search` tool starts.
+ */
+export function formatSearchStatusText(
+  toolName: string,
+  query: string,
+): string {
+  if (toolName !== "web_search") return `Running ${toolName}`;
+  const trimmed = query.trim();
+  if (!trimmed) return "Searching the web";
+  const truncated =
+    trimmed.length > 60 ? trimmed.slice(0, 57) + "..." : trimmed;
+  return `Searching "${truncated}"`;
+}
+
+/**
+ * Status text shown to the client while a web fetch is in flight.
+ * Surfaces the domain so users can tell what page is being read.
+ */
+export function formatFetchStatusText(url: unknown): string {
+  if (typeof url !== "string") return "Reading a page";
+  const domain = extractDomain(url);
+  if (!domain) return "Reading a page";
+  return `Reading ${domain}`;
+}
+
+function computeToolUseStatusText(
+  name: string,
+  input: Record<string, unknown>,
+): string {
+  if (name === "web_search") {
+    const query = typeof input.query === "string" ? input.query : "";
+    return formatSearchStatusText("web_search", query);
+  }
+  if (name === "web_fetch") {
+    return formatFetchStatusText(input.url);
+  }
+  if (
+    name === "skill_execute" &&
+    typeof input.activity === "string" &&
+    input.activity.length > 0
+  ) {
+    return input.activity;
+  }
+  return `Running ${friendlyToolName(name)}`;
+}
+
+function resolveAssistantReplyTimestampTimezone(
+  ctx: AgentLoopConversationContext,
+): string {
+  const config = getConfig();
+  return resolveTurnTimezoneContext({
+    configuredUserTimeZone: config.ui?.userTimezone ?? null,
+    clientTimezone: ctx.clientTimezone ?? null,
+    detectedTimezone: config.ui?.detectedTimezone ?? null,
+    hostTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+  }).effectiveTimezone;
+}
+
 // ── Individual Handlers ──────────────────────────────────────────────
 
 function handleTextDelta(
@@ -403,12 +482,7 @@ export function handleToolUse(
   state.toolCallTimestamps.set(event.id, { startedAt: Date.now() });
   state.currentToolUseId = event.id;
   state.currentTurnToolUseIds.push(event.id);
-  const statusText =
-    event.name === "skill_execute" &&
-    typeof event.input.activity === "string" &&
-    event.input.activity.length > 0
-      ? event.input.activity
-      : `Running ${friendlyToolName(event.name)}`;
+  const statusText = computeToolUseStatusText(event.name, event.input);
   deps.ctx.emitActivityState(
     "tool_running",
     "tool_use_start",
@@ -675,6 +749,7 @@ export function handleToolResult(
     approvalMode: event.approvalMode,
     approvalReason: event.approvalReason,
     riskThreshold: event.riskThreshold,
+    activityMetadata: event.activityMetadata,
   });
 }
 
@@ -983,11 +1058,20 @@ export async function handleMessageComplete(
     const channelId = deps.ctx.trustContext?.requesterChatId;
     if (channelId) {
       const threadTs = getThreadTs(deps.ctx.conversationId);
+      const timestampTimezone = resolveAssistantReplyTimestampTimezone(
+        deps.ctx,
+      );
+      const timestampTimezoneLabel = formatSlackTimezoneLabel(
+        timestampTimezone,
+        { nowMs: state.turnStartedAt },
+      );
       const partialSlackMeta: Partial<SlackMessageMetadata> = {
         source: "slack",
         eventKind: "message",
         channelId,
         ...(threadTs ? { threadTs } : {}),
+        timestampTimezone,
+        ...(timestampTimezoneLabel ? { timestampTimezoneLabel } : {}),
       };
       assistantChannelMetadata.slackMeta = writeSlackMetadata(
         // `channelTs` is filled in by the post-send reconciliation step in
@@ -1166,6 +1250,52 @@ function handleUsage(
   state.llmCallStartedEmitted = false;
 }
 
+/**
+ * Persist a provider-rejected LLM call as an `llm_request_logs` row.
+ *
+ * Mirrors `handleUsage`'s recording side-effect for the failure path: the
+ * loop only reaches the success branch (and emits `usage`) when the
+ * provider returns a response, so without this handler a rejected call
+ * leaves nothing in the inspector — only a pino line saying "The AI
+ * provider rejected the request." The row's `messageId` is left null
+ * here and linked via one of two backfill paths, depending on how the
+ * turn unwinds:
+ *
+ *   - Multi-call turn where a later call also produces a real assistant
+ *     response: `handleMessageComplete` -> `backfillMessageIdOnLogs`
+ *     sweeps this row with the rest, same as a successful-call row.
+ *   - Pure provider-failure turn (no real assistant response): the
+ *     synthetic error-message branch in `conversation-agent-loop.ts`
+ *     persists a stand-in assistant message and calls
+ *     `backfillMessageIdOnLogs` itself, since `message_complete` is
+ *     never emitted on that path. Closing the orphan window inside the
+ *     same synchronous turn prevents a later turn's sweep from wrong-
+ *     attaching this row to an unrelated assistant message.
+ *
+ * Failures inside the recording itself are logged and swallowed — this
+ * mirrors `handleUsage`'s non-fatal stance so a DB hiccup never escalates
+ * a provider rejection into a dispatcher-level throw.
+ */
+function handleProviderError(
+  deps: EventHandlerDeps,
+  event: Extract<AgentEvent, { type: "provider_error" }>,
+): void {
+  try {
+    recordRequestLog(
+      deps.ctx.conversationId,
+      JSON.stringify(event.rawRequest),
+      JSON.stringify(buildProviderErrorResponsePayload(event.error)),
+      undefined,
+      event.actualProvider,
+    );
+  } catch (err) {
+    deps.rlog.warn(
+      { err },
+      "Failed to persist provider-error LLM request log (non-fatal)",
+    );
+  }
+}
+
 // ── Dispatcher ───────────────────────────────────────────────────────
 
 /** Routes an AgentEvent to the appropriate handler. */
@@ -1198,10 +1328,9 @@ export async function dispatchAgentEvent(
         handleToolResult(state, deps, event);
         break;
       case "server_tool_start": {
-        const friendlyNames: Record<string, string> = {
-          web_search: "Searching the web",
-        };
-        const statusText = friendlyNames[event.name] ?? `Running ${event.name}`;
+        const query =
+          typeof event.input.query === "string" ? event.input.query : "";
+        const statusText = formatSearchStatusText(event.name, query);
         deps.ctx.emitActivityState(
           "tool_running",
           "tool_use_start",
@@ -1209,6 +1338,8 @@ export async function dispatchAgentEvent(
           deps.reqId,
           statusText,
         );
+        state.serverToolStartedAt.set(event.toolUseId, Date.now());
+        state.serverToolInputs.set(event.toolUseId, event.input);
         deps.onEvent({
           type: "tool_use_start",
           toolName: event.name,
@@ -1227,19 +1358,65 @@ export async function dispatchAgentEvent(
           "Thinking",
         );
 
-        // Format web search results into a human-readable string for the client.
-        let resultText = "";
-        if (Array.isArray(event.content) && event.content.length > 0) {
-          resultText = (event.content as unknown[])
-            .filter(
-              (r): r is { type: string; title: string; url: string } =>
-                typeof r === "object" &&
-                r != null &&
-                (r as { type?: string }).type === "web_search_result",
-            )
-            .map((r) => `${r.title}\n${r.url}`)
-            .join("\n\n");
-        }
+        // Prefer `resolvedInput` (Anthropic's accumulated server-tool input,
+        // populated on content_block_stop) over the input captured at
+        // server_tool_start, which is `{}` on Anthropic until the deltas land.
+        const inputForCall =
+          event.resolvedInput ?? state.serverToolInputs.get(event.toolUseId);
+        const query =
+          typeof inputForCall?.query === "string" ? inputForCall.query : "";
+        const startedAt =
+          state.serverToolStartedAt.get(event.toolUseId) ?? Date.now();
+        const durationMs = Date.now() - startedAt;
+        state.serverToolStartedAt.delete(event.toolUseId);
+        state.serverToolInputs.delete(event.toolUseId);
+
+        const rawBlocks = Array.isArray(event.content) ? event.content : [];
+        const results: WebSearchResultItem[] = rawBlocks
+          .filter(
+            (r): r is { type: string; title: string; url: string } =>
+              typeof r === "object" &&
+              r != null &&
+              (r as { type?: string }).type === "web_search_result",
+          )
+          .map((r, i) => {
+            const domain = extractDomain(r.url);
+            return {
+              rank: i + 1,
+              title: r.title,
+              url: r.url,
+              domain,
+              faviconUrl: faviconUrlForDomain(domain),
+              // snippet intentionally absent — Anthropic native content is encrypted/opaque
+            };
+          });
+
+        // Only Anthropic produces structured `web_search_tool_result` blocks
+        // that map cleanly onto `WebSearchMetadata` (provider-tagged
+        // "anthropic-native"). Other providers (e.g. OpenAI's responses
+        // `web_search_call`) share this event channel but their results are
+        // woven into the text stream — emitting "anthropic-native" metadata
+        // for them would mis-label the provider and ship empty results.
+        const isAnthropicNative = deps.ctx.provider.name === "anthropic";
+
+        const errorMessage = event.isError
+          ? (event.errorMessage ?? event.errorCode ?? "Search failed")
+          : undefined;
+
+        const metadata: WebSearchMetadata | undefined = isAnthropicNative
+          ? {
+              query,
+              provider: "anthropic-native",
+              resultCount: results.length,
+              durationMs,
+              results,
+              ...(errorMessage ? { errorMessage } : {}),
+            }
+          : undefined;
+
+        const resultText = results
+          .map((r) => `${r.title}\n${r.url}`)
+          .join("\n\n");
 
         deps.onEvent({
           type: "tool_result",
@@ -1248,18 +1425,49 @@ export async function dispatchAgentEvent(
           isError: event.isError,
           conversationId: deps.ctx.conversationId,
           toolUseId: event.toolUseId,
+          ...(metadata ? { activityMetadata: { webSearch: metadata } } : {}),
         });
         break;
       }
       case "error":
         handleError(state, deps, event);
         break;
+      case "provider_error":
+        handleProviderError(deps, event);
+        break;
       case "message_complete":
         await handleMessageComplete(state, deps, event);
         break;
       case "usage":
         handleUsage(state, deps, event);
         break;
+      case "agent_loop_exit":
+        // Stamp the exit reason onto the most-recent llm_request_logs
+        // row for this conversation. The final `usage` event of the run
+        // lands its row immediately before this event arrives (in the
+        // normal-dispatch path; the wake path handles ordering
+        // explicitly via `pendingExitReason`).
+        //
+        // Wrapped in try/catch so a DB hiccup here can't tear down the
+        // surrounding dispatch — the outer try/catch already swallows
+        // errors, but logging here gives the diagnosis hook a clear
+        // attribution to the exit handler specifically.
+        try {
+          setAgentLoopExitReasonOnLatestLog(
+            deps.ctx.conversationId,
+            event.reason,
+          );
+        } catch (err) {
+          log.warn(
+            {
+              err,
+              conversationId: deps.ctx.conversationId,
+              reason: event.reason,
+            },
+            "Failed to persist agent_loop_exit_reason (non-fatal)",
+          );
+        }
+        break;
     }
   } catch (err) {
     log.error(