@vellumai/assistant 0.6.4 → 0.6.5

package/src/__tests__/slack-skill.test.ts

@@ -4,12 +4,7 @@ import { describe, expect, test } from "bun:test";
 
 import type { SlackConversation } from "../messaging/providers/slack/types.js";
 
-const BUNDLED_SKILLS_DIR = join(
-  import.meta.dir,
-  "..",
-  "config",
-  "bundled-skills",
-);
+const REPO_SKILLS_DIR = join(import.meta.dir, "..", "..", "..", "skills");
 
 describe("slack adapter isPrivate mapping", () => {
   // Inline the mapping logic to test it independently of the adapter module
@@ -47,7 +42,7 @@ describe("slack adapter isPrivate mapping", () => {
 });
 
 describe("slack skill has no TOOLS.json (uses Web API via CLI)", () => {
-  const toolsPath = join(BUNDLED_SKILLS_DIR, "slack", "TOOLS.json");
+  const toolsPath = join(REPO_SKILLS_DIR, "slack", "TOOLS.json");
 
   test("TOOLS.json does not exist", () => {
     expect(() => readFileSync(toolsPath)).toThrow();
@@ -56,7 +51,7 @@ describe("slack skill has no TOOLS.json (uses Web API via CLI)", () => {
 
 describe("slack skill SKILL.md", () => {
   const skillMd = readFileSync(
-    join(BUNDLED_SKILLS_DIR, "slack", "SKILL.md"),
+    join(REPO_SKILLS_DIR, "slack", "SKILL.md"),
     "utf-8",
   );
 

package/src/__tests__/starter-bundle.test.ts

@@ -2,6 +2,8 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 
+import { ruleScope } from "@vellumai/ces-contracts";
+
 // Set up a temp directory before importing trust-store
 const TEST_ROOT = join(
   import.meta.dirname ?? __dirname,
@@ -135,4 +137,37 @@ describe("Starter approval bundle", () => {
       expect(defaultIds.has(starterRule.id)).toBe(false);
     }
   });
+
+  test("accepted starter rules have canonical persisted shape after disk round-trip", () => {
+    acceptStarterBundle();
+
+    // Force a disk round-trip by clearing the cache
+    clearCache();
+    const allRules = getAllRules();
+    const starterRuleIds = new Set(getStarterBundleRules().map((r) => r.id));
+    const starterRules = allRules.filter((r) => starterRuleIds.has(r.id));
+
+    expect(starterRules.length).toBe(getStarterBundleRules().length);
+
+    for (const rule of starterRules) {
+      // Effective scope is always "everywhere" — scoped and generic tools
+      // carry an explicit scope field, while URL tools omit it (ruleScope
+      // returns "everywhere" for rules without a scope field).
+      expect(ruleScope(rule)).toBe("everywhere");
+
+      // Starter rules are all allow decisions — they should not carry
+      // family-specific metadata signals (executionTarget).
+      // These rules cover read-only/information-gathering tools that are
+      // either generic or scoped family, but none require high-risk access.
+      expect(rule.executionTarget).toBeUndefined();
+
+      // Base fields must be present
+      expect(rule.id).toBeTruthy();
+      expect(rule.tool).toBeTruthy();
+      expect(rule.pattern).toBeTruthy();
+      expect(rule.decision).toBe("allow");
+      expect(rule.priority).toBeGreaterThan(0);
+      expect(rule.createdAt).toBeGreaterThan(0);
+    }
+  });
 });

package/src/__tests__/subagent-call-site-routing.test.ts

@@ -0,0 +1,280 @@
+/**
+ * Regression test for the subagent provider routing fix.
+ *
+ * Before the fix, `SubagentManager.spawn()` constructed the Conversation with
+ * `getProvider(appConfig.llm.default.provider)` directly, which meant per-call
+ * `llm.callSites.subagentSpawn.provider` overrides only changed the request
+ * *metadata* the downstream client saw — the actual HTTP transport still
+ * belonged to `llm.default.provider`. After the fix, the provider is wrapped
+ * in `CallSiteRoutingProvider`, which consults the resolver per call and
+ * routes to the resolved provider's transport when it differs from the
+ * default.
+ *
+ * This test stubs the `Conversation` constructor and the provider registry
+ * so we can capture the provider that `SubagentManager` passes into
+ * `Conversation`, then verify it's a `CallSiteRoutingProvider` that selects
+ * the right transport for the `subagentSpawn` callSite.
+ */
+import { describe, expect, mock, test } from "bun:test";
+
+import type { ServerMessage } from "../daemon/message-protocol.js";
+
+// Capture the provider passed to Conversation.
+let capturedProvider: unknown = undefined;
+
+// Stub Conversation so spawn() doesn't try to actually run an agent loop —
+// we only care about what provider it was constructed with.
+class FakeConversation {
+  constructor(
+    _id: string,
+    provider: unknown,
+    _systemPrompt: string,
+    _maxTokens: number,
+    _sendToClient: (msg: ServerMessage) => void,
+  ) {
+    capturedProvider = provider;
+  }
+  updateClient() {}
+  setIsSubagent() {}
+  hasSystemPromptOverride = false;
+  setSubagentAllowedTools() {}
+  preactivateSkills() {}
+  preactivateSkillsAsync() {}
+  setSpawnHints() {}
+  injectInheritedContext() {}
+  setActiveBranchId() {}
+  setBranchTag() {}
+  setForkPolicy() {}
+  setForkParentMessageCount() {}
+  setForkParentSystemPrompt() {}
+  enqueueMessage() {
+    return { rejected: false, queued: true };
+  }
+  abort() {}
+  dispose() {}
+  messages = [];
+  usageStats = { inputTokens: 0, outputTokens: 0, estimatedCost: 0 };
+  sendToClient() {}
+  loadFromDb() {
+    return Promise.resolve();
+  }
+  persistUserMessage() {
+    return "msg-id";
+  }
+  runAgentLoop() {
+    return Promise.resolve();
+  }
+  getCurrentSystemPrompt() {
+    return "system";
+  }
+}
+
+mock.module("../daemon/conversation.js", () => ({
+  Conversation: FakeConversation,
+}));
+
+mock.module("../memory/conversation-bootstrap.js", () => ({
+  bootstrapConversation: () => ({ id: "conv-id" }),
+}));
+
+mock.module("../prompts/system-prompt.js", () => ({
+  buildSystemPrompt: () => "system prompt",
+  buildSubagentSystemPrompt: () => "subagent system",
+}));
+
+// Provider registry — return distinct stubs so we can verify the selection.
+const anthropicStub = { name: "anthropic" };
+const openaiStub = { name: "openai" };
+
+mock.module("../providers/registry.js", () => ({
+  getProvider: (name: string) => {
+    if (name === "anthropic") return anthropicStub;
+    if (name === "openai") return openaiStub;
+    throw new Error(`unknown provider: ${name}`);
+  },
+}));
+
+// Mutable LLM config — tests rewrite this per-case.
+let mockLlmConfig: Record<string, unknown> = {};
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    llm: mockLlmConfig,
+    rateLimit: { maxRequestsPerMinute: 0 },
+  }),
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+// ── Imports (after mocks) ───────────────────────────────────────────────────
+
+import { LLMSchema } from "../config/schemas/llm.js";
+import { CallSiteRoutingProvider } from "../providers/call-site-routing.js";
+import { SubagentManager } from "../subagent/manager.js";
+
+function setLlmConfig(raw: unknown): void {
+  mockLlmConfig = LLMSchema.parse(raw) as Record<string, unknown>;
+}
+
+describe("SubagentManager — provider call-site routing", () => {
+  test("wraps the default provider in CallSiteRoutingProvider", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "claude-opus-4-7" },
+    });
+
+    capturedProvider = undefined;
+    const manager = new SubagentManager();
+    await manager.spawn(
+      {
+        parentConversationId: "parent-1",
+        label: "test",
+        objective: "test objective",
+      },
+      () => {},
+    );
+
+    expect(capturedProvider).toBeInstanceOf(CallSiteRoutingProvider);
+  });
+
+  test("the wrapped provider exposes the default provider's name (stable identity for outer wrappers)", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "claude-opus-4-7" },
+      callSites: {
+        subagentSpawn: { provider: "openai", model: "gpt-5.4" },
+      },
+    });
+
+    capturedProvider = undefined;
+    const manager = new SubagentManager();
+    await manager.spawn(
+      {
+        parentConversationId: "parent-2",
+        label: "test",
+        objective: "test objective",
+      },
+      () => {},
+    );
+
+    // The wrapper exposes the *default* provider's name (so wrappers further
+    // out — e.g. RateLimitProvider — see a stable identity), but routes the
+    // actual sendMessage to the resolved provider. The routing behavior
+    // itself is exercised in the next describe block with a fully-stubbed
+    // provider pair.
+    expect(capturedProvider).toBeInstanceOf(CallSiteRoutingProvider);
+    const wrapper = capturedProvider as CallSiteRoutingProvider;
+    expect(wrapper.name).toBe("anthropic");
+  });
+
+  test("falls back to default provider when subagentSpawn callSite is absent", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "claude-opus-4-7" },
+      // No subagentSpawn override.
+    });
+
+    capturedProvider = undefined;
+    const manager = new SubagentManager();
+    await manager.spawn(
+      {
+        parentConversationId: "parent-3",
+        label: "test",
+        objective: "test objective",
+      },
+      () => {},
+    );
+
+    expect(capturedProvider).toBeInstanceOf(CallSiteRoutingProvider);
+    // Default provider's name surfaces.
+    expect((capturedProvider as { name: string }).name).toBe("anthropic");
+  });
+});
+
+// ── Direct unit test for CallSiteRoutingProvider's selection logic ─────────
+
+describe("CallSiteRoutingProvider — selectProvider behavior", () => {
+  test("routes to the resolved provider when callSite.provider differs from default", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "claude-opus-4-7" },
+      callSites: {
+        subagentSpawn: { provider: "openai", model: "gpt-5.4" },
+      },
+    });
+
+    let calledOnDefault = false;
+    let calledOnAlternative = false;
+
+    const defaultProvider = {
+      name: "anthropic",
+      sendMessage: async () => {
+        calledOnDefault = true;
+        return {
+          content: [],
+          model: "anthropic",
+          usage: { inputTokens: 0, outputTokens: 0 },
+          stopReason: "end_turn" as const,
+        };
+      },
+    };
+    const altProvider = {
+      name: "openai",
+      sendMessage: async () => {
+        calledOnAlternative = true;
+        return {
+          content: [],
+          model: "openai",
+          usage: { inputTokens: 0, outputTokens: 0 },
+          stopReason: "end_turn" as const,
+        };
+      },
+    };
+
+    const wrapper = new CallSiteRoutingProvider(defaultProvider, (name) => {
+      if (name === "openai") return altProvider;
+      return undefined;
+    });
+
+    await wrapper.sendMessage([], undefined, undefined, {
+      config: { callSite: "subagentSpawn" },
+    });
+
+    expect(calledOnAlternative).toBe(true);
+    expect(calledOnDefault).toBe(false);
+  });
+
+  test("routes to default when no callSite provided", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "claude-opus-4-7" },
+      callSites: {
+        subagentSpawn: { provider: "openai", model: "gpt-5.4" },
+      },
+    });
+
+    let calledOnDefault = false;
+
+    const defaultProvider = {
+      name: "anthropic",
+      sendMessage: async () => {
+        calledOnDefault = true;
+        return {
+          content: [],
+          model: "anthropic",
+          usage: { inputTokens: 0, outputTokens: 0 },
+          stopReason: "end_turn" as const,
+        };
+      },
+    };
+
+    const wrapper = new CallSiteRoutingProvider(
+      defaultProvider,
+      () => undefined,
+    );
+
+    await wrapper.sendMessage([], undefined, undefined, {
+      config: {},
+    });
+
+    expect(calledOnDefault).toBe(true);
+  });
+});

package/src/__tests__/suggestion-routes.test.ts

@@ -383,7 +383,164 @@ describe("GET /v1/suggestion", () => {
     expect(body.suggestion).toBeNull();
   });
 
-  test("uses latency-optimized model intent", async () => {
+  test("strips leaked <reply> tags from LLM response", async () => {
+    // Realistic output when the model re-emits </reply> before the stop_sequence
+    // catches, or when a leading <reply> slips in.
+    const provider = makeMockProvider("sounds good</reply>");
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-asst-1",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([{ type: "text", text: "Want to go?" }]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    const res = await handleGetSuggestion(url, deps);
+    const body = (await res.json()) as { suggestion: string };
+
+    expect(body.suggestion).toBe("sounds good");
+  });
+
+  test("passes prefill message, stop_sequences, and max_tokens", async () => {
+    const provider = makeMockProvider("on my way");
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-user-1",
+        conversationId: "conv-test",
+        role: "user",
+        content: JSON.stringify([{ type: "text", text: "heading out" }]),
+        createdAt: Date.now() - 1000,
+        metadata: null,
+      },
+      {
+        id: "msg-asst-shape",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([
+          { type: "text", text: "see you there — which door?" },
+        ]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    await handleGetSuggestion(url, deps);
+
+    expect(provider.sendMessage).toHaveBeenCalledTimes(1);
+    const callArgs = provider.sendMessage.mock.calls[0] as unknown[];
+    const messages = callArgs[0] as Array<{
+      role: string;
+      content: Array<{ type: string; text: string }>;
+    }>;
+    const options = callArgs[3] as {
+      config?: {
+        stop_sequences?: string[];
+        max_tokens?: number;
+      };
+    };
+
+    expect(messages.length).toBe(2);
+    expect(messages[1].role).toBe("assistant");
+    expect(messages[1].content[0].text).toBe("<reply>");
+    expect(options.config?.stop_sequences).toEqual(["</reply>"]);
+    expect(options.config?.max_tokens).toBe(60);
+    expect(messages[0].content[0].text).toContain("<assistant_message>");
+    expect(messages[0].content[0].text).toContain("<user_message>");
+  });
+
+  test("includes prior user message in prompt when present", async () => {
+    const provider = makeMockProvider("on my way");
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-user-1",
+        conversationId: "conv-test",
+        role: "user",
+        content: JSON.stringify([
+          { type: "text", text: "running late, should I grab coffee?" },
+        ]),
+        createdAt: Date.now() - 1000,
+        metadata: null,
+      },
+      {
+        id: "msg-asst-1",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([
+          { type: "text", text: "yes please, an americano would be great" },
+        ]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    await handleGetSuggestion(url, deps);
+
+    const callArgs = provider.sendMessage.mock.calls[0] as unknown[];
+    const messages = callArgs[0] as Array<{
+      content: Array<{ text: string }>;
+    }>;
+    const userPrompt = messages[0].content[0].text;
+    expect(userPrompt).toContain(
+      "<user_message>running late, should I grab coffee?</user_message>",
+    );
+    expect(userPrompt).toContain(
+      "<assistant_message>yes please, an americano would be great</assistant_message>",
+    );
+  });
+
+  test("uses placeholder when no prior user message exists", async () => {
+    const provider = makeMockProvider("sure");
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-asst-first",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([{ type: "text", text: "hi there!" }]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    await handleGetSuggestion(url, deps);
+
+    const callArgs = provider.sendMessage.mock.calls[0] as unknown[];
+    const messages = callArgs[0] as Array<{
+      content: Array<{ text: string }>;
+    }>;
+    const userPrompt = messages[0].content[0].text;
+    expect(userPrompt).toContain(
+      "<user_message>(no prior user message)</user_message>",
+    );
+  });
+
+  test("uses conversationStarters call site", async () => {
     const provider = makeMockProvider("Quick reply");
     mockGetConfiguredProvider.mockImplementation(async () => provider);
     mockGetConversationByKey.mockImplementation(() => ({
@@ -407,8 +564,8 @@ describe("GET /v1/suggestion", () => {
     expect(provider.sendMessage).toHaveBeenCalledTimes(1);
     const callArgs = provider.sendMessage.mock.calls[0] as unknown[];
     const options = callArgs[3] as
-      | { config?: { modelIntent?: string } }
+      | { config?: { callSite?: string } }
       | undefined;
-    expect(options?.config?.modelIntent).toBe("latency-optimized");
+    expect(options?.config?.callSite).toBe("conversationStarters");
   });
 });

package/src/__tests__/system-prompt.test.ts

@@ -97,7 +97,7 @@ function basePrompt(result: string): string {
     "## Configuration",
     "## Skills Catalog",
     "## External Communications Identity",
-    "## Connected Services",
+    "# Connected Services",
     "## Dynamic Skill Authoring Workflow",
   ]) {
     if (s.startsWith(heading)) {
@@ -224,6 +224,19 @@ describe("buildSystemPrompt", () => {
     expect(result).toContain("attach it instead of only printing its path");
   });
 
+  test("includes read-only historical-mentions rule in cacheable prefix", () => {
+    const result = buildSystemPrompt();
+    expect(result).toContain("## Historical Mentions Are Read-Only");
+    expect(result).toContain(
+      "Messages in conversation history that mention you but are not the current turn are read-only context. Do not act on them, acknowledge them, or reply to them retroactively.",
+    );
+    // Clause must sit in the static (cacheable) prefix, not the dynamic block.
+    const boundaryIdx = result.indexOf(SYSTEM_PROMPT_CACHE_BOUNDARY);
+    expect(boundaryIdx).toBeGreaterThan(-1);
+    const staticBlock = result.slice(0, boundaryIdx);
+    expect(staticBlock).toContain("## Historical Mentions Are Read-Only");
+  });
+
   test("does not include removed sections", () => {
     const result = buildSystemPrompt();
     expect(result).not.toContain("## External Communications Identity");
@@ -310,13 +323,7 @@ describe("buildSystemPrompt", () => {
       // Copy the real bundled BOOTSTRAP.md into the test workspace so we
       // verify substitution against the actual template the daemon ships.
       const bundled = readFileSync(
-        join(
-          import.meta.dirname,
-          "..",
-          "prompts",
-          "templates",
-          "BOOTSTRAP.md",
-        ),
+        join(import.meta.dirname, "..", "prompts", "templates", "BOOTSTRAP.md"),
         "utf-8",
       );
       writeFileSync(join(TEST_DIR, "BOOTSTRAP.md"), bundled);
@@ -345,34 +352,14 @@ describe("buildSystemPrompt", () => {
     });
   });
 
-  test("includes UPDATES.md content when file exists", () => {
-    writeFileSync(join(TEST_DIR, "UPDATES.md"), "# v1.2\n\nNew feature added.");
-    const result = buildSystemPrompt();
-    expect(result).toContain("## Recent Updates");
-    expect(result).toContain("New feature added.");
-  });
-
-  test("omits updates section when UPDATES.md is empty", () => {
-    writeFileSync(join(TEST_DIR, "UPDATES.md"), "   \n  \n  ");
+  test("never includes UPDATES.md content in system prompt", () => {
+    const updatesBody = "# v1.2\n\nNew feature added. UNIQUE_UPDATES_MARKER.";
+    writeFileSync(join(TEST_DIR, "UPDATES.md"), updatesBody);
     const result = buildSystemPrompt();
     expect(result).not.toContain("## Recent Updates");
-  });
-
-  test("omits updates section when UPDATES.md does not exist", () => {
-    const result = buildSystemPrompt();
-    expect(result).not.toContain("## Recent Updates");
-  });
-
-  test("includes update handling instructions when UPDATES.md exists", () => {
-    writeFileSync(join(TEST_DIR, "UPDATES.md"), "# v1.3\n\nSome update notes.");
-    const result = buildSystemPrompt();
-    expect(result).toContain("### Update Handling");
-    expect(result).toContain("Use your judgment");
-  });
-
-  test("omits update handling instructions when UPDATES.md is absent", () => {
-    const result = buildSystemPrompt();
-    expect(result).not.toContain("### Update Handling");
+    expect(result).not.toContain(updatesBody);
+    expect(result).not.toContain("UNIQUE_UPDATES_MARKER");
+    expect(result).not.toContain("Update Handling");
   });
 
   test("strips comment lines starting with _ from prompt files", () => {
@@ -689,7 +676,7 @@ describe("ensurePromptFiles", () => {
     // SOUL.md and IDENTITY.md are absent (they will be freshly seeded from
     // templates, but onboarding should not re-trigger).
     mkdirSync(join(TEST_DIR, "users"), { recursive: true });
-    writeFileSync(join(TEST_DIR, "users", "sidd.md"), "# Sidd persona");
+    writeFileSync(join(TEST_DIR, "users", "alice.md"), "# Alice persona");
 
     ensurePromptFiles();
 

package/src/__tests__/task-runner.test.ts

@@ -7,6 +7,8 @@ mock.module("../util/logger.js", () => ({
     }),
 }));
 
+import { ruleScope } from "@vellumai/ces-contracts";
+
 import { getDb, initializeDb } from "../memory/db.js";
 import { getTaskRunRules } from "../tasks/ephemeral-permissions.js";
 import { renderTemplate, runTask } from "../tasks/task-runner.js";
@@ -131,7 +133,7 @@ describe("runTask", () => {
     expect(rulesWhileRunning).toHaveLength(2);
     expect(rulesWhileRunning[0].tool).toBe("read_file");
     expect(rulesWhileRunning[1].tool).toBe("write_file");
-    expect(rulesWhileRunning[0].scope).toBe("everywhere");
+    expect(ruleScope(rulesWhileRunning[0])).toBe("everywhere");
     expect(rulesWhileRunning[0].decision).toBe("allow");
     expect(rulesWhileRunning[0].priority).toBe(75);