@vellumai/assistant 0.8.10 → 0.8.11-staging.1

package/src/__tests__/tool-start-timestamp.test.ts

@@ -0,0 +1,218 @@
+/**
+ * Tests for `handleToolUse` stamping a tool start time both on the live
+ * `tool_use_start` event and on the already-durable persisted tool_use block.
+ *
+ * The live event carries `startedAt` so clients can anchor a running
+ * elapsed-time counter to the server clock instead of SSE receipt time. The
+ * persisted `_startedAt` (mapped to the wire `startedAt` in handlers/shared.ts)
+ * lets a `/messages` snapshot fetched mid-tool — e.g. after a refresh or
+ * reconnect, before the live event was seen — still render that counter.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock platform (must precede imports that read it) ─────────────────────────
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+let mockedRowContent = "";
+const updates: Array<{ id: string; content: string }> = [];
+
+mock.module("../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: (id: string) =>
+    mockedRowContent ? { id, content: mockedRowContent } : null,
+  updateMessageContent: (id: string, content: string) => {
+    updates.push({ id, content });
+  },
+  provenanceFromTrustContext: () => ({}),
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
+}));
+
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+mock.module("../runtime/assistant-stream-state.js", () => ({
+  getCurrentSeq: () => 0,
+  recordPersistedSeq: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type { AgentEvent } from "../agent/loop.js";
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  handleToolUse,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import type { ServerMessage } from "../daemon/message-protocol.js";
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function makeDeps(onEvent: (event: ServerMessage) => void): EventHandlerDeps {
+  return {
+    ctx: {
+      conversationId: "test-conv",
+      provider: { name: "anthropic" },
+      traceEmitter: { emit: () => {} },
+      streamThinking: false,
+      emitActivityState: () => {},
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent,
+    reqId: "test-req",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as unknown as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "web",
+      assistantMessageInterface: "web",
+    } as unknown as EventHandlerDeps["turnInterfaceContext"],
+    applyCompaction: async () => {},
+  };
+}
+
+function toolUseEvent(
+  id: string,
+  name: string,
+): Extract<AgentEvent, { type: "tool_use" }> {
+  return { type: "tool_use", id, name, input: { command: "ls" } };
+}
+
+function findBlockById(
+  rawContent: string,
+  id: string,
+): Record<string, unknown> {
+  const parsed = JSON.parse(rawContent) as Array<Record<string, unknown>>;
+  const block = parsed.find((b) => b.id === id);
+  if (!block) throw new Error(`block ${id} not found`);
+  return block;
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("handleToolUse — tool start timestamp", () => {
+  beforeEach(() => {
+    updates.length = 0;
+    mockedRowContent = "";
+  });
+
+  test("emits a numeric startedAt on the tool_use_start event and records it in state", () => {
+    // GIVEN a fresh handler state with a persisted assistant message
+    const toolUseId = "tu_bash";
+    const state: EventHandlerState = createEventHandlerState();
+    state.lastAssistantMessageId = "msg-1";
+    mockedRowContent = JSON.stringify([
+      {
+        type: "tool_use",
+        id: toolUseId,
+        name: "bash",
+        input: { command: "ls" },
+      },
+    ]);
+    const events: ServerMessage[] = [];
+    const before = Date.now();
+
+    // WHEN the tool begins
+    handleToolUse(
+      state,
+      makeDeps((e) => events.push(e)),
+      toolUseEvent(toolUseId, "bash"),
+    );
+
+    // THEN the emitted tool_use_start carries a numeric startedAt within the
+    // execution window AND the same value is recorded in toolCallTimestamps
+    const startEvent = events.find((e) => e.type === "tool_use_start");
+    expect(startEvent?.type).toBe("tool_use_start");
+    const startedAt =
+      startEvent?.type === "tool_use_start" ? startEvent.startedAt : undefined;
+    expect(typeof startedAt).toBe("number");
+    expect(startedAt).toBeGreaterThanOrEqual(before);
+    expect(startedAt).toBeLessThanOrEqual(Date.now());
+    expect(state.toolCallTimestamps.get(toolUseId)?.startedAt).toBe(startedAt);
+  });
+
+  test("stamps _startedAt onto the in-flight persisted tool_use block", () => {
+    // GIVEN a durable tool_use block with no timing metadata yet
+    const toolUseId = "tu_bash";
+    const state: EventHandlerState = createEventHandlerState();
+    state.lastAssistantMessageId = "msg-1";
+    mockedRowContent = JSON.stringify([
+      {
+        type: "tool_use",
+        id: toolUseId,
+        name: "bash",
+        input: { command: "ls" },
+      },
+    ]);
+    const events: ServerMessage[] = [];
+
+    // WHEN the tool begins
+    handleToolUse(
+      state,
+      makeDeps((e) => events.push(e)),
+      toolUseEvent(toolUseId, "bash"),
+    );
+
+    // THEN the persisted block is updated with _startedAt matching the event,
+    // so a mid-tool snapshot surfaces the start time via handlers/shared.ts
+    expect(updates).toHaveLength(1);
+    const block = findBlockById(updates[0].content, toolUseId);
+    const startEvent = events.find((e) => e.type === "tool_use_start");
+    const startedAt =
+      startEvent?.type === "tool_use_start" ? startEvent.startedAt : undefined;
+    expect(block._startedAt).toBe(startedAt);
+  });
+
+  test("does not write when no persisted block matches the tool id", () => {
+    // GIVEN a persisted message whose only block is unrelated to the tool
+    const state: EventHandlerState = createEventHandlerState();
+    state.lastAssistantMessageId = "msg-1";
+    mockedRowContent = JSON.stringify([{ type: "text", text: "thinking" }]);
+
+    // WHEN a tool begins
+    handleToolUse(
+      state,
+      makeDeps(() => {}),
+      toolUseEvent("tu_missing", "bash"),
+    );
+
+    // THEN no persisted-content write is attempted
+    expect(updates).toHaveLength(0);
+  });
+});

package/src/__tests__/tools-get-route.test.ts

@@ -0,0 +1,202 @@
+/**
+ * Tests for the `tools` route handler (operationId `tools_get`), which backs
+ * both the macOS permission-simulator catalog (`names`/`schemas`) and the
+ * `assistant tools list` CLI command (`tools`). The `tools` array carries
+ * every registered tool with its description, author-asserted risk band,
+ * category, and the source (core / skill / plugin / mcp) that contributed it
+ * — with the source read from the registry's ownership map rather than off
+ * the tool object.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type { Conversation } from "../daemon/conversation.js";
+import {
+  clearConversations,
+  setConversation,
+} from "../daemon/conversation-registry.js";
+import { RiskLevel } from "../permissions/types.js";
+import { ROUTES } from "../runtime/routes/settings-routes.js";
+import {
+  __resetRegistryForTesting,
+  registerMcpTools,
+  registerPluginTools,
+  registerSkillTools,
+  registerTool,
+} from "../tools/registry.js";
+import type { Tool, ToolContext, ToolExecutionResult } from "../tools/types.js";
+
+interface ToolListEntry {
+  name: string;
+  description: string;
+  riskLevel: string;
+  category: string;
+  source: string;
+}
+
+interface ToolsGetResponse {
+  names: string[];
+  schemas: Record<string, unknown>;
+  tools: ToolListEntry[];
+}
+
+function makeFakeTool(name: string, extras: Partial<Tool> = {}): Tool {
+  return {
+    name,
+    description: `Fake ${name}`,
+    defaultRiskLevel: RiskLevel.Low,
+    executionTarget: "sandbox",
+    input_schema: { type: "object", properties: {}, required: [] },
+    category: "",
+    async execute(
+      _input: Record<string, unknown>,
+      _context: ToolContext,
+    ): Promise<ToolExecutionResult> {
+      return { content: "ok", isError: false };
+    },
+    ...extras,
+  };
+}
+
+const handler = (() => {
+  const route = ROUTES.find(
+    (r) => r.endpoint === "tools" && r.method === "GET",
+  );
+  if (!route) {
+    throw new Error("No route found for GET tools");
+  }
+  return route.handler;
+})();
+
+/**
+ * Minimal Conversation stand-in exposing only the tool-snapshot accessor the
+ * route reads, registered into the live conversation registry by id.
+ */
+function registerFakeConversation(id: string, toolNames: string[]): void {
+  setConversation(id, {
+    getRegisteredToolNames: () => new Set(toolNames),
+  } as unknown as Conversation);
+}
+
+describe("GET /tools", () => {
+  beforeEach(() => {
+    __resetRegistryForTesting();
+    clearConversations();
+  });
+
+  test("returns each registered tool's metadata, sorted by name, with core source", async () => {
+    // GIVEN two core tools registered out of alphabetical order
+    registerTool(
+      makeFakeTool("b_core_tool", {
+        description: "Beta tool",
+        defaultRiskLevel: RiskLevel.High,
+        category: "testing",
+      }),
+    );
+    registerTool(makeFakeTool("a_core_tool"));
+
+    // WHEN the tools handler runs
+    const { names, schemas, tools } = (await handler({})) as ToolsGetResponse;
+
+    // THEN the catalog fields the permission simulator reads are present
+    expect(names).toContain("a_core_tool");
+    expect(names).toContain("b_core_tool");
+    expect(schemas.a_core_tool).toBeDefined();
+
+    // AND the metadata array is sorted by name with full per-tool metadata
+    const toolNames = tools.map((t) => t.name);
+    expect(toolNames.indexOf("a_core_tool")).toBeLessThan(
+      toolNames.indexOf("b_core_tool"),
+    );
+    expect([...toolNames]).toEqual(
+      [...toolNames].sort((a, b) => a.localeCompare(b)),
+    );
+
+    // AND each core tool reports "core" as its source plus its metadata
+    const beta = tools.find((t) => t.name === "b_core_tool");
+    expect(beta).toEqual({
+      name: "b_core_tool",
+      description: "Beta tool",
+      riskLevel: RiskLevel.High,
+      category: "testing",
+      source: "core",
+    });
+  });
+
+  test("reports plugin, mcp, and skill tools as <kind>:<id>", async () => {
+    // GIVEN one tool from each non-core source
+    registerPluginTools("echo", [makeFakeTool("p_tool")]);
+    registerMcpTools("linear", [makeFakeTool("m_tool")]);
+    registerSkillTools("my-skill", [makeFakeTool("s_tool")]);
+
+    // WHEN the tools handler runs
+    const { tools } = (await handler({})) as ToolsGetResponse;
+
+    // THEN each tool's source encodes its owning extension
+    const sourceOf = (name: string) =>
+      tools.find((t) => t.name === name)?.source;
+    expect(sourceOf("p_tool")).toBe("plugin:echo");
+    expect(sourceOf("m_tool")).toBe("mcp:linear");
+    expect(sourceOf("s_tool")).toBe("skill:my-skill");
+  });
+
+  test("with conversationId, scopes to the conversation's tool snapshot and resolves metadata from the registry", async () => {
+    // GIVEN three core tools in the global registry
+    registerTool(makeFakeTool("a_core_tool"));
+    registerTool(
+      makeFakeTool("b_core_tool", {
+        description: "Beta tool",
+        defaultRiskLevel: RiskLevel.High,
+        category: "testing",
+      }),
+    );
+    registerTool(makeFakeTool("c_core_tool"));
+
+    // AND a conversation whose most-recent-turn snapshot saw only two of them
+    registerFakeConversation("conv-1", ["c_core_tool", "a_core_tool"]);
+
+    // WHEN the handler runs with that conversationId
+    const { names, schemas, tools } = (await handler({
+      queryParams: { conversationId: "conv-1" },
+    })) as ToolsGetResponse;
+
+    // THEN only the conversation's tools are returned, sorted, with schemas
+    expect(names).toEqual(["a_core_tool", "c_core_tool"]);
+    expect(tools.map((t) => t.name)).toEqual(["a_core_tool", "c_core_tool"]);
+    expect(schemas.a_core_tool).toBeDefined();
+    expect(schemas.b_core_tool).toBeUndefined();
+
+    // AND each entry's metadata is resolved from the global registry
+    expect(tools.find((t) => t.name === "a_core_tool")?.source).toBe("core");
+  });
+
+  test("with conversationId, marks a snapshot tool absent from the registry as unknown", async () => {
+    // GIVEN a conversation snapshot referencing a since-unloaded skill tool
+    registerFakeConversation("conv-1", ["ghost_skill_tool"]);
+
+    // WHEN the handler runs with that conversationId
+    const { tools } = (await handler({
+      queryParams: { conversationId: "conv-1" },
+    })) as ToolsGetResponse;
+
+    // THEN the tool still appears, with unknown source and empty metadata
+    expect(tools).toEqual([
+      {
+        name: "ghost_skill_tool",
+        description: "",
+        riskLevel: "unknown",
+        category: "",
+        source: "unknown",
+      },
+    ]);
+  });
+
+  test("with an unknown conversationId, throws a 404 RouteError", () => {
+    // GIVEN no conversation registered under the queried id
+    // WHEN the handler runs with a missing conversationId
+    // THEN it throws an actionable not-found error
+    expect(() =>
+      handler({ queryParams: { conversationId: "missing" } }),
+    ).toThrow(/No active conversation "missing"/);
+  });
+});