@vellumai/assistant 0.9.0 → 0.10.0-dev.202606192226.8ed8a6a

package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts

@@ -1,14 +1,14 @@
 /**
- * Verifies that the `memory-v3-live` cache-anchor signal surfaces on every
- * `SendMessageOptions.config` the loop emits. When the flag is on, the latest
+ * Verifies that the memory-v3-live cache-anchor signal surfaces on every
+ * `SendMessageOptions.config` the loop emits. When v3-live is on, the latest
  * user message carries a volatile `<memory>` block, so the loop sets
  * `providerConfig.mutableLatestUserMessage` to tell the provider to anchor its
  * long-TTL cache breakpoint on the most recent STABLE message instead.
  *
- * The loop reads the flag directly where it assembles `providerConfig`, so the
- * signal is sourced from the flag rather than a run option. When the flag is
- * off the field is omitted (not `false`/`undefined`) so the wire stays
- * byte-identical to today.
+ * The loop reads `config.memory.v3.live` directly where it assembles
+ * `providerConfig`, so the signal is sourced from config rather than a run
+ * option. When v3-live is off the field is omitted (not `false`/`undefined`) so
+ * the wire stays byte-identical to today.
  */
 
 import { afterEach, describe, expect, mock, test } from "bun:test";
@@ -19,11 +19,14 @@ mock.module("../util/logger.js", () => ({
   getLogger: () => makeMockLogger(),
 }));
 
+// AgentLoop reads the v3-live gate (`config.memory.v3.live`) via
+// `isMemoryV3Live` to decide the cache-anchor signal; drive it per-test.
+let memoryV3LiveSlot = false;
+mock.module("../config/memory-v3-gate.js", () => ({
+  isMemoryV3Live: () => memoryV3LiveSlot,
+}));
+
 import { AgentLoop } from "../agent/loop.js";
-import {
-  clearCachedOverrides,
-  setCachedOverrides,
-} from "../config/feature-flag-cache.js";
 import type {
   Message,
   Provider,
@@ -82,12 +85,12 @@ function makeRecordingProvider(responses: ProviderResponse[]): {
 
 describe("AgentLoop.run — mutableLatestUserMessage from memory-v3-live", () => {
   afterEach(() => {
-    clearCachedOverrides();
+    memoryV3LiveSlot = false;
   });
 
   test("sets mutableLatestUserMessage on every LLM call when memory-v3-live is on (multi-turn)", async () => {
     // GIVEN memory-v3-live is enabled
-    setCachedOverrides({ "memory-v3-live": true }, { fromGateway: true });
+    memoryV3LiveSlot = true;
 
     // AND a provider that records the config of each LLM call across a tool round-trip
     const { provider, configs } = makeRecordingProvider([
@@ -131,7 +134,7 @@ describe("AgentLoop.run — mutableLatestUserMessage from memory-v3-live", () =>
 
   test("omits mutableLatestUserMessage when memory-v3-live is off (flag-off byte-identity)", async () => {
     // GIVEN memory-v3-live is off (no override; registry default is false)
-    clearCachedOverrides();
+    memoryV3LiveSlot = false;
 
     // AND a provider that records the config of each LLM call
     const { provider, configs } = makeRecordingProvider([textResponse("hi")]);

package/src/__tests__/agent-loop-output-hooks.test.ts

@@ -365,6 +365,75 @@ describe("agent loop output hooks", () => {
     expect(calls[0].systemPrompt).toContain("[EDITED]");
   });
 
+  test("pre-model-call routes the call to the hook's chosen inference profile", async () => {
+    // GIVEN a hook that selects an inference profile for the user-facing call
+    registerOutputHookPlugin({
+      preModelCall: (ctx) => {
+        if (ctx.callSite !== "mainAgent") return;
+        ctx.modelProfile = "fast-profile";
+      },
+    });
+    const { provider, calls } = createMockProvider([textResponse("hi")]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+
+    // WHEN the loop issues the user-facing provider call
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: collect([]),
+      callSite: "mainAgent",
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN the provider call carries the hook's profile as the override
+    expect(calls[0].options?.config?.overrideProfile).toBe("fast-profile");
+  });
+
+  test("pre-model-call seeds modelProfile from the resolved override and clearing it drops the override", async () => {
+    // GIVEN a hook that observes the seeded override and then clears it
+    let seeded: string | null | undefined;
+    registerOutputHookPlugin({
+      preModelCall: (ctx) => {
+        seeded = ctx.modelProfile;
+        ctx.modelProfile = null;
+      },
+    });
+    const { provider, calls } = createMockProvider([textResponse("hi")]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+
+    // WHEN the loop runs with an ad-hoc override profile
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: collect([]),
+      callSite: "mainAgent",
+      overrideProfile: "seed-profile",
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN the hook saw the loop's resolved override
+    expect(seeded).toBe("seed-profile");
+    // AND clearing it sends no override on the provider call
+    expect(calls[0].options?.config?.overrideProfile).toBeUndefined();
+  });
+
+  // Context-window sizing and overflow recovery read the profile resolved
+  // before the hook runs, so a hook routing to a smaller-window profile does
+  // not resize the budget gate for that call. Tracked for a follow-up that
+  // resolves the routed profile ahead of the context-sizing gate.
+  test.todo(
+    "pre-model-call routing resizes the context-window budget for the routed profile",
+    () => {},
+  );
+
   test("fail-open: a hook that mutates in place AND then throws cannot corrupt the persisted content", async () => {
     // The hook mutates the array it receives before throwing. If the loop
     // handed the hook the real `assistantMessage.content` array, the

package/src/__tests__/agent-loop-override-profile.test.ts

@@ -195,6 +195,7 @@ interface CapturedRunAgentLoopOptions {
   titleText?: string;
   callSite?: string;
   overrideProfile?: string;
+  forceOverrideProfile?: boolean;
 }
 
 const capturedRunAgentLoopOptions: CapturedRunAgentLoopOptions[] = [];
@@ -323,6 +324,29 @@ describe("SubagentManager.spawn — overrideProfile inheritance", () => {
     const captured = capturedRunAgentLoopOptions[0];
     expect(captured.callSite).toBe("subagentSpawn");
     expect(captured.overrideProfile).toBe("fast");
+    expect("forceOverrideProfile" in captured).toBe(false);
+  });
+
+  test("forwards forced overrideProfile from SubagentConfig into runAgentLoop", async () => {
+    capturedRunAgentLoopOptions.length = 0;
+
+    const manager = new SubagentManager();
+    await manager.spawn(
+      {
+        parentConversationId: "parent-forced",
+        label: "child",
+        objective: "do the thing",
+        overrideProfile: "fast",
+        forceOverrideProfile: true,
+      },
+      () => {},
+    );
+
+    expect(capturedRunAgentLoopOptions).toHaveLength(1);
+    const captured = capturedRunAgentLoopOptions[0];
+    expect(captured.callSite).toBe("subagentSpawn");
+    expect(captured.overrideProfile).toBe("fast");
+    expect(captured.forceOverrideProfile).toBe(true);
   });
 
   test("omits overrideProfile when SubagentConfig does not set it", async () => {
@@ -344,6 +368,7 @@ describe("SubagentManager.spawn — overrideProfile inheritance", () => {
     // Field must be absent rather than carrying `undefined`, mirroring the
     // agent loop's "field omitted when unset" contract.
     expect("overrideProfile" in captured).toBe(false);
+    expect("forceOverrideProfile" in captured).toBe(false);
   });
 });
 

package/src/__tests__/agent-loop-pushes-post-hook-prompt.test.ts

@@ -0,0 +1,306 @@
+/**
+ * Tests for the `system_prompt_changed` event emitted by AgentLoop when the
+ * `pre-model-call` hook chain mutates the system prompt.
+ *
+ * The event fires whenever the post-hook `systemPrompt` differs from the
+ * value the loop handed the hook (compared against `providerOptions.systemPrompt`,
+ * not the context object — which the hook may mutate in place).
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Stub pipes ──────────────────────────────────────────────────────
+// Same logger-stub shape as the rest of the test suite — pino pulls are
+// agent-loop-time too, but we silence them so the test stays deterministic.
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+// Mock plugin pipeline. Two modes:
+//  - return mode (default): returns `hookResponse` as the hook result.
+//  - in-place mode: mutates ctx.systemPrompt directly and returns the same
+//    object, simulating a hook that edits the context without returning a
+//    new value — the scenario that requires comparing against
+//    providerOptions.systemPrompt, not preModelCtx.systemPrompt.
+type PreModelCallCtx = {
+  conversationId: string;
+  callSite: string | null;
+  systemPrompt: string | null;
+  modelProfile: string | null;
+  deferAssistantOutput: boolean;
+  logger: unknown;
+};
+let hookResponse: PreModelCallCtx = {
+  conversationId: "test-conv",
+  callSite: "mainAgent",
+  systemPrompt:
+    "BASELINE-PROMPT\n\n<!-- advisor:steering -->\nAPPENDED-FROM-HOOK",
+  modelProfile: null,
+  deferAssistantOutput: false,
+  logger: {},
+};
+let hookMutatesInPlace = false;
+mock.module("../plugins/pipeline.js", () => ({
+  runHook: async (
+    _hook: unknown,
+    ctx: PreModelCallCtx,
+  ): Promise<PreModelCallCtx> => {
+    if (hookMutatesInPlace) {
+      ctx.systemPrompt = hookResponse.systemPrompt;
+      return ctx;
+    }
+    return hookResponse;
+  },
+}));
+
+// Minimal `getConfig` so the loop's defaults/loader path doesn't trip.
+let mockLlmConfig: Record<string, unknown> = {};
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ llm: mockLlmConfig }),
+}));
+
+import type { AgentEvent } from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import { LLMSchema } from "../config/schemas/llm.js";
+import type { TrustContext } from "../daemon/trust-context.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+} from "../providers/types.js";
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "hi" }],
+};
+
+const trust: TrustContext = {
+  sourceChannel: "vellum",
+  trustClass: "guardian",
+};
+
+function setLlmConfig(raw: unknown): void {
+  mockLlmConfig = LLMSchema.parse(raw) as Record<string, unknown>;
+}
+
+beforeEach(() => {
+  mockLlmConfig = LLMSchema.parse({}) as Record<string, unknown>;
+  hookMutatesInPlace = false;
+  hookResponse = {
+    conversationId: "test-conv",
+    callSite: "mainAgent",
+    systemPrompt:
+      "BASELINE-PROMPT\n\n<!-- advisor:steering -->\nAPPENDED-FROM-HOOK",
+    modelProfile: null,
+    deferAssistantOutput: false,
+    logger: {},
+  };
+});
+
+/**
+ * Provider that records the system prompt it sees on every send and short-
+ * circuits the turn after one call. We don't need streaming for verifying
+ * prompt pushback.
+ */
+function makeRecordingProvider(): {
+  provider: Provider;
+  capturedSystemPrompts: () => string[];
+} {
+  const captured: string[] = [];
+  let callCount = 0;
+  const provider: Provider = {
+    name: "anthropic",
+    async sendMessage(
+      _messages: Message[],
+      options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      if (options?.systemPrompt !== undefined) {
+        captured.push(options.systemPrompt);
+      }
+      callCount += 1;
+      return {
+        content: [{ type: "text", text: callCount === 1 ? "done" : "..." }],
+        model: "mock-model",
+        usage: { inputTokens: 1, outputTokens: 1 },
+        stopReason: "end_turn",
+      };
+    },
+  };
+  return { provider, capturedSystemPrompts: () => captured };
+}
+
+describe("AgentLoop system_prompt_changed event", () => {
+  test("fires when the hook mutates the prompt on mainAgent", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    const mutatedPrompt = "BASELINE\n\n<!-- advisor:steering -->";
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "mainAgent",
+      systemPrompt: mutatedPrompt,
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "BASELINE",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "mainAgent",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(1);
+    expect(changedEvents[0].systemPrompt).toBe(mutatedPrompt);
+  });
+
+  test("fires on non-mainAgent call sites too", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "subagentSpawn",
+      systemPrompt: "MUTATED-BY-HOOK",
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "BASELINE",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "subagentSpawn",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(1);
+    expect(changedEvents[0].systemPrompt).toBe("MUTATED-BY-HOOK");
+  });
+
+  test("does not fire when hook returns the same prompt", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "mainAgent",
+      systemPrompt: "SAME-PROMPT",
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "SAME-PROMPT",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "mainAgent",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(0);
+  });
+
+  test("fires when the hook mutates ctx.systemPrompt in place", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    // In-place mode: the mock mutates ctx.systemPrompt directly and
+    // returns the same object reference. The old comparison against
+    // preModelCtx.systemPrompt would miss this because preModelCtx IS
+    // the mutated object. Comparing against providerOptions.systemPrompt
+    // (captured pre-hook) catches it.
+    hookMutatesInPlace = true;
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "mainAgent",
+      systemPrompt: "BASELINE\n\n<!-- advisor:steering -->",
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "BASELINE",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "mainAgent",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(1);
+    expect(changedEvents[0].systemPrompt).toBe(
+      "BASELINE\n\n<!-- advisor:steering -->",
+    );
+  });
+});

package/src/__tests__/always-loaded-tools-guard.test.ts

@@ -28,7 +28,7 @@ afterAll(() => {
 });
 
 describe("always-loaded tool count", () => {
-  test("should be exactly 11 with recall occupying the existing slot", async () => {
+  test("should be exactly 10 with recall occupying the existing slot", async () => {
     await initializeTools();
     const allDefs = getAllToolDefinitions();
 
@@ -52,7 +52,6 @@ describe("always-loaded tool count", () => {
     // path would allow unchecked host command execution.
     const expectedNames = [
       "bash",
-      "credential_store",
       "file_edit",
       "file_read",
       "file_write",
@@ -66,6 +65,6 @@ describe("always-loaded tool count", () => {
 
     expect(activeNames).toEqual(expectedNames);
     expect(activeNames.filter((name) => name === "recall")).toHaveLength(1);
-    expect(activeTools.length).toBe(11);
+    expect(activeTools.length).toBe(10);
   });
 });

package/src/__tests__/app-compiler.test.ts

@@ -2,9 +2,17 @@ import { existsSync } from "node:fs";
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import {
+  afterAll,
+  beforeAll,
+  describe,
+  expect,
+  setDefaultTimeout,
+  test,
+} from "bun:test";
 
 import { compileApp } from "../bundler/app-compiler.js";
+import { ensureCompilerTools } from "../bundler/compiler-tools.js";
 import {
   ALLOWED_PACKAGES,
   getCacheDir,
@@ -19,8 +27,14 @@ import {
 
 let tempDir: string;
 
+// Compiler-tool download (esbuild + preact from npm) can take 10s+ on a cold
+// CI cache. Raise the file-level default so beforeAll isn't killed mid-install.
+// Each test file runs in its own process, so this doesn't leak.
+setDefaultTimeout(30_000);
+
 beforeAll(async () => {
   tempDir = await mkdtemp(join(tmpdir(), "app-compiler-test-"));
+  await ensureCompilerTools();
 });
 
 afterAll(async () => {

package/src/__tests__/app-dir-path-guard.test.ts

@@ -20,7 +20,6 @@ const ALLOWLIST = new Set([
   "assistant/src/memory/app-git-service.ts", // uses getAppsDir for git repo root, not per-app paths
   "assistant/src/daemon/app-source-watcher.ts", // uses getAppsDir for recursive fs.watch root, not per-app paths
   "assistant/src/tools/filesystem/write.ts", // uses getAppsDir as an exemption root for the artifact-HTML guard, not per-app paths
-  "assistant/src/memory/preloaded-apps.ts", // seeds fixed-slug apps (id === dirName by construction) and writes the definition JSON, which app-store has no fixed-id helper for
 ]);
 
 function isTestFile(filePath: string): boolean {

package/src/__tests__/approval-cascade.test.ts

@@ -170,6 +170,12 @@ mock.module("../memory/retriever.js", () => ({
 
 mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
   ContextWindowManager: class {
+    estimateInputTokens() {
+      return 0;
+    }
+    get tokenCountInputs() {
+      return { systemPrompt: "", tools: undefined };
+    }
     constructor() {}
     updateConfig() {}
     shouldCompact() {