@vellumai/assistant 0.10.2 → 0.10.3-dev.202606252046.9075fd5

package/src/__tests__/workspace-tool-loader.test.ts

@@ -26,7 +26,7 @@
  *   and skipped (no silent provider-safe rewrite — operator must rename).
  * - Multiple workspace tools register in a single batch.
  */
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, rmSync, utimesSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, beforeEach, describe, expect, test } from "bun:test";
@@ -42,7 +42,10 @@ import {
   registerTool,
 } from "../tools/registry.js";
 import type { Tool, ToolContext, ToolExecutionResult } from "../tools/types.js";
-import { loadWorkspaceTools } from "../tools/workspace-tools/loader.js";
+import {
+  __resetWorkspaceToolCacheForTesting,
+  loadWorkspaceTools,
+} from "../tools/workspace-tools/loader.js";
 
 // Per-test counter so each writeTool() call lands in a unique tempdir,
 // defeating bun's per-URL ESM cache between tests. Without this, a
@@ -87,6 +90,23 @@ function writeRemovedSentinel(name: string): void {
   writeFileSync(join(toolsDir, `${name}.removed`), "");
 }
 
+/** Delete `<workspaceDir>/tools/<name><ext>` (defaults to `.ts`). */
+function removeToolFile(name: string, ext = ".ts"): void {
+  rmSync(join(currentWorkspaceDir, "tools", `${name}${ext}`), { force: true });
+}
+
+/**
+ * Overwrite an existing tool file and bump its mtime into the future so the
+ * reconcile's mtime gate re-imports it even when the rewrite lands within
+ * the same millisecond as the original write.
+ */
+function rewriteTool(name: string, body: string, ext = ".ts"): void {
+  const path = join(currentWorkspaceDir, "tools", `${name}${ext}`);
+  writeFileSync(path, body);
+  const future = new Date(Date.now() + 5000);
+  utimesSync(path, future, future);
+}
+
 function makeFakeCoreTool(name: string): Tool {
   return {
     name,
@@ -94,6 +114,9 @@ function makeFakeCoreTool(name: string): Tool {
     category: "test",
     defaultRiskLevel: RiskLevel.Low,
     executionTarget: "sandbox",
+    // Match the finalized shape the registry stores (defaults filled), so
+    // `getCoreToolOverride(name)` toEqual comparisons hold after registration.
+    exclusive: false,
     input_schema: { type: "object", properties: {}, required: [] },
     async execute(
       _input: Record<string, unknown>,
@@ -133,6 +156,7 @@ export default {
 describe("workspace tool loader", () => {
   beforeEach(() => {
     __clearRegistryForTesting();
+    __resetWorkspaceToolCacheForTesting();
     freshWorkspace();
   });
 
@@ -316,4 +340,173 @@ export default 42;
     const names = getWorkspaceToolNames().sort();
     expect(names).toEqual(["alpha", "beta", "gamma"]);
   });
+
+  // ── Reconcile-on-read behavior ─────────────────────────────────────────
+  //
+  // loadWorkspaceTools() is idempotent and re-derives registry state from
+  // disk on every call. These cases cover the deltas a repeat call applies,
+  // which is what replaces the old filesystem watcher.
+
+  test("repeat call with no disk changes is a no-op (does not throw or duplicate)", async () => {
+    writeTool("stable_tool", WELL_FORMED_BODY);
+
+    await loadWorkspaceTools();
+    // A second reconcile must not throw on the already-registered name —
+    // the mtime cache recognizes the unchanged file and skips re-import.
+    await loadWorkspaceTools();
+
+    expect(getTool("stable_tool")).toBeDefined();
+    expect(getWorkspaceToolNames()).toEqual(["stable_tool"]);
+  });
+
+  test("a file added after the first reconcile registers on the next", async () => {
+    writeTool("first", WELL_FORMED_BODY);
+    await loadWorkspaceTools();
+    expect(getWorkspaceToolNames()).toEqual(["first"]);
+
+    writeTool("second", WELL_FORMED_BODY);
+    await loadWorkspaceTools();
+
+    expect(getWorkspaceToolNames().sort()).toEqual(["first", "second"]);
+  });
+
+  test("a changed file is re-imported on the next reconcile", async () => {
+    writeTool("mutable", WELL_FORMED_BODY);
+    await loadWorkspaceTools();
+    expect(getTool("mutable")?.description).toBe("from workspace");
+
+    rewriteTool(
+      "mutable",
+      `
+export default {
+  description: "edited in place",
+  defaultRiskLevel: "low",
+  input_schema: { type: "object", properties: {}, required: [] },
+  async execute() {
+    return { content: "edited", isError: false };
+  },
+};
+`,
+    );
+    await loadWorkspaceTools();
+
+    expect(getTool("mutable")?.description).toBe("edited in place");
+  });
+
+  test("a deleted net-new tool file is unregistered on the next reconcile", async () => {
+    writeTool("ephemeral", WELL_FORMED_BODY);
+    await loadWorkspaceTools();
+    expect(getTool("ephemeral")).toBeDefined();
+
+    removeToolFile("ephemeral");
+    await loadWorkspaceTools();
+
+    expect(getTool("ephemeral")).toBeUndefined();
+    expect(getWorkspaceToolNames()).toEqual([]);
+  });
+
+  test("deleting an override file restores the stashed core tool", async () => {
+    const core = makeFakeCoreTool("restore_me");
+    registerTool(core);
+    writeTool("restore_me", WELL_FORMED_BODY);
+
+    await loadWorkspaceTools();
+    expect(getToolOwner("restore_me")?.kind).toBe("workspace");
+
+    removeToolFile("restore_me");
+    await loadWorkspaceTools();
+
+    expect(getToolOwner("restore_me")).toBeUndefined();
+    expect(getTool("restore_me")).toEqual(core);
+    expect(getCoreToolOverride("restore_me")).toBeUndefined();
+  });
+
+  test("deleting a .removed sentinel restores the stripped core tool", async () => {
+    const core = makeFakeCoreTool("strip_then_restore");
+    registerTool(core);
+    writeRemovedSentinel("strip_then_restore");
+
+    await loadWorkspaceTools();
+    expect(getTool("strip_then_restore")).toBeUndefined();
+    expect(getStrippedCoreToolNames()).toContain("strip_then_restore");
+
+    removeToolFile("strip_then_restore", ".removed");
+    await loadWorkspaceTools();
+
+    expect(getTool("strip_then_restore")).toEqual(core);
+    expect(getStrippedCoreToolNames()).not.toContain("strip_then_restore");
+  });
+
+  test("the registered name is the filename stem, ignoring the file's own name field", async () => {
+    // The default export sets a different `name` — the loader must pin the
+    // registered name to the stem ("stem_wins") so the mtime cache and the
+    // unregister-on-delete path stay keyed by the same name.
+    writeTool(
+      "stem_wins",
+      `
+export default {
+  name: "different_name",
+  description: "name field should be ignored",
+  defaultRiskLevel: "low",
+  input_schema: { type: "object", properties: {}, required: [] },
+  async execute() {
+    return { content: "ok", isError: false };
+  },
+};
+`,
+    );
+
+    await loadWorkspaceTools();
+
+    expect(getTool("stem_wins")).toBeDefined();
+    expect(getTool("different_name")).toBeUndefined();
+    expect(getWorkspaceToolNames()).toEqual(["stem_wins"]);
+
+    // Deleting the file unregisters by stem — no leaked "different_name".
+    removeToolFile("stem_wins");
+    await loadWorkspaceTools();
+    expect(getTool("stem_wins")).toBeUndefined();
+    expect(getTool("different_name")).toBeUndefined();
+  });
+
+  test("per-tool isolation on reconcile: a bad file does not drop a valid edited tool", async () => {
+    writeTool("good_edit", WELL_FORMED_BODY);
+    await loadWorkspaceTools();
+    expect(getTool("good_edit")?.description).toBe("from workspace");
+
+    // Add a file that throws at import, and edit the good tool, in the same
+    // reconcile. The broken file must not prevent the edited tool from
+    // re-registering.
+    writeTool("broken_now", `throw new Error("boom at import");`);
+    rewriteTool(
+      "good_edit",
+      `
+export default {
+  description: "edited and still here",
+  defaultRiskLevel: "low",
+  input_schema: { type: "object", properties: {}, required: [] },
+  async execute() {
+    return { content: "ok", isError: false };
+  },
+};
+`,
+    );
+    await loadWorkspaceTools();
+
+    expect(getTool("broken_now")).toBeUndefined();
+    expect(getTool("good_edit")?.description).toBe("edited and still here");
+  });
+
+  test("an edit that breaks an existing tool keeps the prior registration", async () => {
+    writeTool("was_good", WELL_FORMED_BODY);
+    await loadWorkspaceTools();
+    expect(getTool("was_good")?.description).toBe("from workspace");
+
+    // Rewrite the file into something that throws at import. The prior,
+    // working registration must stay in place rather than being torn down.
+    rewriteTool("was_good", `throw new Error("now broken");`);
+    await loadWorkspaceTools();
+
+    expect(getTool("was_good")?.description).toBe("from workspace");
+  });
 });

package/src/a2a/__tests__/e2e-a2a-channel.test.ts

@@ -42,6 +42,7 @@ mock.module("../../util/logger.js", () => ({
 // the a2a.enabled flag. We use the real config system backed by initializeDb's
 // workspace directory.
 
+import { seedContactChannel } from "../../__tests__/helpers/seed-contact-channel.js";
 import {
   invalidateConfigCache,
   loadRawConfig,
@@ -80,6 +81,15 @@ const originalFetch = globalThis.fetch;
 // Helpers
 // ---------------------------------------------------------------------------
 
+/** Read a channel's local ACL columns directly to assert the gateway dual-write. */
+function aclColumns(
+  channelId: string,
+): { status: string; policy: string } | null {
+  return getSqlite()
+    .query("SELECT status, policy FROM contact_channels WHERE id = ?")
+    .get(channelId) as { status: string; policy: string } | null;
+}
+
 function resetTables(): void {
   const sqlite = getSqlite();
   sqlite.run("DELETE FROM a2a_tasks");
@@ -164,17 +174,16 @@ describe("e2e: trusted contact setup", () => {
         {
           type: "a2a",
           address: "assistant-b",
-          status: "active",
-          policy: "allow",
         },
       ],
     });
 
+    // upsertContact persists the a2a channel identity; the gateway owns the ACL
+    // status verdict.
     const contact = findContactByAddress("a2a", "assistant-b");
     expect(contact).not.toBeNull();
     expect(contact!.channels.some((ch) => ch.type === "a2a")).toBe(true);
     const a2aChannel = contact!.channels.find((ch) => ch.type === "a2a");
-    expect(a2aChannel!.status).toBe("active");
     expect(a2aChannel!.address).toBe("assistant-b");
   });
 });
@@ -355,21 +364,13 @@ describe("e2e: unknown sender blocked (ACL enforcement)", () => {
   });
 
   test("trusted contact exists with active a2a channel — ACL passes", async () => {
-    const { upsertContact } = await import("../../contacts/contact-store.js");
-
     // Pre-create a trusted contact for the sender
-    upsertContact({
+    seedContactChannel({
+      sourceChannel: "a2a",
+      externalUserId: "trusted-assistant",
       displayName: "Trusted Bot",
-      contactType: "assistant",
-      role: "contact",
-      channels: [
-        {
-          type: "a2a",
-          address: "trusted-assistant",
-          status: "active",
-          policy: "allow",
-        },
-      ],
+      status: "active",
+      policy: "allow",
     });
 
     // Verify the contact exists (the ACL check the runtime performs)
@@ -378,8 +379,9 @@ describe("e2e: unknown sender blocked (ACL enforcement)", () => {
 
     const a2aChannel = contact!.channels.find((ch) => ch.type === "a2a");
     expect(a2aChannel).toBeTruthy();
-    expect(a2aChannel!.status).toBe("active");
-    expect(a2aChannel!.policy).toBe("allow");
+    const acl = aclColumns(a2aChannel!.id);
+    expect(acl!.status).toBe("active");
+    expect(acl!.policy).toBe("allow");
 
     // A task from this sender would pass the ACL check
     const msg = makeRequestMessage();
@@ -391,28 +393,21 @@ describe("e2e: unknown sender blocked (ACL enforcement)", () => {
   });
 
   test("contact exists but channel is blocked — ACL would reject", async () => {
-    const { upsertContact } = await import("../../contacts/contact-store.js");
-
-    upsertContact({
+    seedContactChannel({
+      sourceChannel: "a2a",
+      externalUserId: "blocked-assistant",
       displayName: "Blocked Bot",
-      contactType: "assistant",
-      role: "contact",
-      channels: [
-        {
-          type: "a2a",
-          address: "blocked-assistant",
-          status: "blocked",
-          policy: "deny",
-        },
-      ],
+      status: "blocked",
+      policy: "deny",
     });
 
     const contact = findContactByAddress("a2a", "blocked-assistant");
     expect(contact).not.toBeNull();
 
     const a2aChannel = contact!.channels.find((ch) => ch.type === "a2a");
-    expect(a2aChannel!.status).toBe("blocked");
-    expect(a2aChannel!.policy).toBe("deny");
+    const acl = aclColumns(a2aChannel!.id);
+    expect(acl!.status).toBe("blocked");
+    expect(acl!.policy).toBe("deny");
   });
 });
 
@@ -507,26 +502,19 @@ describe("e2e: full A2A round-trip", () => {
     setConfigEnabled(true);
 
     // Step 1: Create trusted contact for Assistant B (platform-mediated)
-    upsertContact({
+    seedContactChannel({
+      sourceChannel: "a2a",
+      externalUserId: "assistant-b",
       displayName: "Assistant B",
-      contactType: "assistant",
-      role: "contact",
-      channels: [
-        {
-          type: "a2a",
-          address: "assistant-b",
-          status: "active",
-          policy: "allow",
-        },
-      ],
+      status: "active",
+      policy: "allow",
     });
 
     // Step 2: Verify trusted contact was created
     const contact = findContactByAddress("a2a", "assistant-b");
     expect(contact).not.toBeNull();
-    expect(contact!.channels.find((ch) => ch.type === "a2a")!.status).toBe(
-      "active",
-    );
+    const a2aChannel = contact!.channels.find((ch) => ch.type === "a2a")!;
+    expect(aclColumns(a2aChannel.id)!.status).toBe("active");
 
     // Step 3: Simulate inbound A2A message from B (as if B sent us a request)
     const inboundMsg = makeRequestMessage({

package/src/agent/loop-exclusive-tool.test.ts

@@ -0,0 +1,154 @@
+/**
+ * Verifies the agent loop's exclusive-tool dispatch: when a tool the loop is
+ * told is exclusive appears in a multi-call turn, only that tool runs and the
+ * siblings are deferred un-run with a benign result — so the model incorporates
+ * the exclusive tool's output before acting on anything else. Drives the REAL
+ * loop, mocking only the provider boundary.
+ */
+import { describe, expect, test } from "bun:test";
+
+import { createMockProvider } from "../__tests__/helpers/mock-provider.js";
+import type { ContentBlock, ProviderResponse } from "../providers/types.js";
+import { AgentLoop } from "./loop.js";
+
+const endTurn = (text: string): ProviderResponse => ({
+  content: [{ type: "text", text }],
+  model: "mock-model",
+  usage: { inputTokens: 1, outputTokens: 1 },
+  stopReason: "end_turn",
+});
+
+const toolUseTurn = (
+  blocks: Array<{ id: string; name: string }>,
+): ProviderResponse => ({
+  content: [
+    { type: "text", text: "working" },
+    ...blocks.map((b) => ({
+      type: "tool_use" as const,
+      id: b.id,
+      name: b.name,
+      input: {},
+    })),
+  ],
+  model: "mock-model",
+  usage: { inputTokens: 1, outputTokens: 1 },
+  stopReason: "tool_use",
+});
+
+function toolResults(history: { content: ContentBlock[] }[]) {
+  return history
+    .flatMap((m) => m.content)
+    .filter(
+      (b): b is Extract<ContentBlock, { type: "tool_result" }> =>
+        b.type === "tool_result",
+    );
+}
+
+const baseRun = {
+  requestId: "req-excl",
+  onEvent: () => {},
+  callSite: "mainAgent" as const,
+  trust: { sourceChannel: "vellum" as const, trustClass: "unknown" as const },
+};
+
+describe("AgentLoop — exclusive tool deferral", () => {
+  test("runs the exclusive tool alone and defers sibling calls un-run", async () => {
+    const { provider } = createMockProvider([
+      toolUseTurn([
+        { id: "call-exclusive", name: "exclusive_tool" },
+        { id: "call-edit", name: "write_file" },
+      ]),
+      endTurn("done"),
+    ]);
+
+    const executed: string[] = [];
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "sys",
+      conversationId: "excl-1",
+      tools: [
+        {
+          name: "exclusive_tool",
+          description: "",
+          input_schema: { type: "object" },
+        },
+        {
+          name: "write_file",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async (name) => {
+        executed.push(name);
+        return { content: `ran ${name}`, isError: false };
+      },
+      isExclusiveTool: (name) => name === "exclusive_tool",
+    });
+
+    const { history } = await loop.run({
+      ...baseRun,
+      messages: [{ role: "user", content: [{ type: "text", text: "do it" }] }],
+    });
+
+    // Only the exclusive tool actually executed.
+    expect(executed).toEqual(["exclusive_tool"]);
+
+    const results = toolResults(history);
+    const exclusiveResult = results.find(
+      (b) => b.tool_use_id === "call-exclusive",
+    )!;
+    const editResult = results.find((b) => b.tool_use_id === "call-edit")!;
+
+    // The exclusive tool ran; the sibling came back un-run (not an error) so the
+    // model can re-issue it after reading the guidance.
+    expect(exclusiveResult.content).toBe("ran exclusive_tool");
+    expect(editResult.content).toContain("not run");
+    expect(editResult.content).toContain("exclusive_tool");
+    expect(editResult.is_error).toBe(false);
+  });
+
+  test("runs sibling tools normally when no exclusive tool is present", async () => {
+    const { provider } = createMockProvider([
+      toolUseTurn([
+        { id: "call-read", name: "read_file" },
+        { id: "call-write", name: "write_file" },
+      ]),
+      endTurn("done"),
+    ]);
+
+    const executed: string[] = [];
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "sys",
+      conversationId: "excl-2",
+      tools: [
+        {
+          name: "read_file",
+          description: "",
+          input_schema: { type: "object" },
+        },
+        {
+          name: "write_file",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async (name) => {
+        executed.push(name);
+        return { content: `ran ${name}`, isError: false };
+      },
+      isExclusiveTool: (name) => name === "exclusive_tool",
+    });
+
+    const { history } = await loop.run({
+      ...baseRun,
+      messages: [{ role: "user", content: [{ type: "text", text: "do it" }] }],
+    });
+
+    // Both non-exclusive tools ran; nothing was deferred.
+    expect(executed.sort()).toEqual(["read_file", "write_file"]);
+    for (const result of toolResults(history)) {
+      expect(result.content).not.toContain("not run");
+    }
+  });
+});