@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -142,7 +142,6 @@ function fakeCompaction(
 ): { trust: TrustContext } {
   createContextWindowManager({
     provider: { name: "mock-provider" } as unknown as Provider,
-    systemPrompt: "system",
     config: {} as unknown as ContextWindowConfig,
     conversationId,
   });

package/src/__tests__/agent-loop-pushes-post-hook-prompt.test.ts

@@ -0,0 +1,306 @@
+/**
+ * Tests for the `system_prompt_changed` event emitted by AgentLoop when the
+ * `pre-model-call` hook chain mutates the system prompt.
+ *
+ * The event fires whenever the post-hook `systemPrompt` differs from the
+ * value the loop handed the hook (compared against `providerOptions.systemPrompt`,
+ * not the context object — which the hook may mutate in place).
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Stub pipes ──────────────────────────────────────────────────────
+// Same logger-stub shape as the rest of the test suite — pino pulls are
+// agent-loop-time too, but we silence them so the test stays deterministic.
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+// Mock plugin pipeline. Two modes:
+//  - return mode (default): returns `hookResponse` as the hook result.
+//  - in-place mode: mutates ctx.systemPrompt directly and returns the same
+//    object, simulating a hook that edits the context without returning a
+//    new value — the scenario that requires comparing against
+//    providerOptions.systemPrompt, not preModelCtx.systemPrompt.
+type PreModelCallCtx = {
+  conversationId: string;
+  callSite: string | null;
+  systemPrompt: string | null;
+  modelProfile: string | null;
+  deferAssistantOutput: boolean;
+  logger: unknown;
+};
+let hookResponse: PreModelCallCtx = {
+  conversationId: "test-conv",
+  callSite: "mainAgent",
+  systemPrompt:
+    "BASELINE-PROMPT\n\n<!-- advisor:steering -->\nAPPENDED-FROM-HOOK",
+  modelProfile: null,
+  deferAssistantOutput: false,
+  logger: {},
+};
+let hookMutatesInPlace = false;
+mock.module("../plugins/pipeline.js", () => ({
+  runHook: async (
+    _hook: unknown,
+    ctx: PreModelCallCtx,
+  ): Promise<PreModelCallCtx> => {
+    if (hookMutatesInPlace) {
+      ctx.systemPrompt = hookResponse.systemPrompt;
+      return ctx;
+    }
+    return hookResponse;
+  },
+}));
+
+// Minimal `getConfig` so the loop's defaults/loader path doesn't trip.
+let mockLlmConfig: Record<string, unknown> = {};
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ llm: mockLlmConfig }),
+}));
+
+import type { AgentEvent } from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import { LLMSchema } from "../config/schemas/llm.js";
+import type { TrustContext } from "../daemon/trust-context.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+} from "../providers/types.js";
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "hi" }],
+};
+
+const trust: TrustContext = {
+  sourceChannel: "vellum",
+  trustClass: "guardian",
+};
+
+function setLlmConfig(raw: unknown): void {
+  mockLlmConfig = LLMSchema.parse(raw) as Record<string, unknown>;
+}
+
+beforeEach(() => {
+  mockLlmConfig = LLMSchema.parse({}) as Record<string, unknown>;
+  hookMutatesInPlace = false;
+  hookResponse = {
+    conversationId: "test-conv",
+    callSite: "mainAgent",
+    systemPrompt:
+      "BASELINE-PROMPT\n\n<!-- advisor:steering -->\nAPPENDED-FROM-HOOK",
+    modelProfile: null,
+    deferAssistantOutput: false,
+    logger: {},
+  };
+});
+
+/**
+ * Provider that records the system prompt it sees on every send and short-
+ * circuits the turn after one call. We don't need streaming for verifying
+ * prompt pushback.
+ */
+function makeRecordingProvider(): {
+  provider: Provider;
+  capturedSystemPrompts: () => string[];
+} {
+  const captured: string[] = [];
+  let callCount = 0;
+  const provider: Provider = {
+    name: "anthropic",
+    async sendMessage(
+      _messages: Message[],
+      options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      if (options?.systemPrompt !== undefined) {
+        captured.push(options.systemPrompt);
+      }
+      callCount += 1;
+      return {
+        content: [{ type: "text", text: callCount === 1 ? "done" : "..." }],
+        model: "mock-model",
+        usage: { inputTokens: 1, outputTokens: 1 },
+        stopReason: "end_turn",
+      };
+    },
+  };
+  return { provider, capturedSystemPrompts: () => captured };
+}
+
+describe("AgentLoop system_prompt_changed event", () => {
+  test("fires when the hook mutates the prompt on mainAgent", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    const mutatedPrompt = "BASELINE\n\n<!-- advisor:steering -->";
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "mainAgent",
+      systemPrompt: mutatedPrompt,
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "BASELINE",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "mainAgent",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(1);
+    expect(changedEvents[0].systemPrompt).toBe(mutatedPrompt);
+  });
+
+  test("fires on non-mainAgent call sites too", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "subagentSpawn",
+      systemPrompt: "MUTATED-BY-HOOK",
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "BASELINE",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "subagentSpawn",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(1);
+    expect(changedEvents[0].systemPrompt).toBe("MUTATED-BY-HOOK");
+  });
+
+  test("does not fire when hook returns the same prompt", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "mainAgent",
+      systemPrompt: "SAME-PROMPT",
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "SAME-PROMPT",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "mainAgent",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(0);
+  });
+
+  test("fires when the hook mutates ctx.systemPrompt in place", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "mock-model" },
+    });
+
+    // In-place mode: the mock mutates ctx.systemPrompt directly and
+    // returns the same object reference. The old comparison against
+    // preModelCtx.systemPrompt would miss this because preModelCtx IS
+    // the mutated object. Comparing against providerOptions.systemPrompt
+    // (captured pre-hook) catches it.
+    hookMutatesInPlace = true;
+    hookResponse = {
+      conversationId: "test-conv",
+      callSite: "mainAgent",
+      systemPrompt: "BASELINE\n\n<!-- advisor:steering -->",
+      modelProfile: null,
+      deferAssistantOutput: false,
+      logger: {},
+    };
+
+    const { provider } = makeRecordingProvider();
+    const events: AgentEvent[] = [];
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "BASELINE",
+      config: {},
+      conversationId: "test-conv",
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust,
+      callSite: "mainAgent",
+    });
+
+    const changedEvents = events.filter(
+      (e) => e.type === "system_prompt_changed",
+    );
+    expect(changedEvents).toHaveLength(1);
+    expect(changedEvents[0].systemPrompt).toBe(
+      "BASELINE\n\n<!-- advisor:steering -->",
+    );
+  });
+});

package/src/__tests__/agent-loop-regrowth-guard.test.ts

@@ -101,7 +101,6 @@ function installManager(opts?: { proactiveExhausted?: boolean }): {
   const exhausted = opts?.proactiveExhausted ?? false;
   createContextWindowManager({
     provider: { name: "mock-provider" } as unknown as Provider,
-    systemPrompt: "system",
     config: {} as unknown as ContextWindowConfig,
     conversationId: CONVERSATION_ID,
   });

package/src/__tests__/agent-loop.test.ts

@@ -2327,6 +2327,7 @@ describe("AgentLoop", () => {
       messages: [userMessage],
       onEvent: collectEvents(events),
       trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      callSite: "mainAgent",
     });
 
     // Provider should be called 3 times: initial, empty response, retry
@@ -2410,6 +2411,7 @@ describe("AgentLoop", () => {
       messages: [userMessage],
       onEvent: collectEvents(events),
       trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      callSite: "mainAgent",
     });
 
     // Provider called exactly 2 times: initial [text+tool_use], then empty.
@@ -2477,6 +2479,7 @@ describe("AgentLoop", () => {
       messages: [userMessage],
       onEvent: collectEvents(events),
       trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      callSite: "mainAgent",
     });
 
     // Provider called 3 times: initial, empty, retry (also empty)

package/src/__tests__/agent-wake-override-profile.test.ts

@@ -103,6 +103,8 @@ function makeTarget(): {
     drainQueue: async () => {},
     // Pre-run auto-compaction gate — no-op for these tests.
     maybeCompact: async () => null,
+    buildCurrentSystemPrompt: () => "mock-system-prompt",
+    modelOverride: undefined,
   };
   return { target: target as unknown as Conversation, runArgs };
 }

package/src/__tests__/anthropic-provider.test.ts

@@ -698,10 +698,12 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
   // -----------------------------------------------------------------------
   // mutableLatestUserMessage — volatile trailing user message cache anchor
   // -----------------------------------------------------------------------
-  test("mutableLatestUserMessage: first-of-turn skips the volatile turn-start anchor and anchors the previous stable user message", async () => {
+  test("mutableLatestUserMessage: first-of-turn moves the 1h anchor off the volatile latest message and bridges it with a 5m tail", async () => {
     // The latest user message carries per-turn-volatile content (e.g. an
-    // injected memory block). The long-TTL anchor must move off it onto the
-    // most recent stable user message so the cached prefix stays reusable.
+    // injected memory block). The 1h anchor must move off it onto the most
+    // recent stable user message so the cached prefix stays reusable across
+    // turns; the volatile message still gets a cheap 5m tail breakpoint so the
+    // next request in the same turn has an exact, reachable boundary.
     const messages: Message[] = [
       userMsg("Turn 1"),
       assistantMsg("Response 1"),
@@ -721,13 +723,13 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     }>;
     const userMessages = sent.filter((m) => m.role === "user");
 
-    // Latest (turn-start) user message gets NO long-TTL breakpoint — it's volatile.
+    // Latest (turn-start) user message gets the cheap 5m tail bridge, never the
+    // 1h anchor — it's volatile.
     const latest = userMessages[userMessages.length - 1];
-    for (const block of latest.content) {
-      expect(block.cache_control).toBeUndefined();
-    }
+    const latestLast = latest.content[latest.content.length - 1];
+    expect(latestLast.cache_control).toEqual({ type: "ephemeral", ttl: "5m" });
 
-    // Previous stable user message (Turn 1) becomes the primary anchor.
+    // Previous stable user message (Turn 1) becomes the primary 1h anchor.
     const prev = userMessages[userMessages.length - 2];
     const prevLast = prev.content[prev.content.length - 1];
     expect(prevLast.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
@@ -803,11 +805,143 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       }>;
     }>;
     // Only user message is volatile and there is no prior stable one, so no
-    // long-TTL breakpoint lands on any user message — graceful, no throw.
+    // 1h anchor and no 5m tail bridge land on any user message — there is no
+    // previous-turn anchor to bridge to. Graceful, no throw.
     const lastBlock = sent[0].content[sent[0].content.length - 1];
     expect(lastBlock.cache_control).toBeUndefined();
   });
 
+  test("mutableLatestUserMessage: first-of-turn after a long autonomous tail bridges the volatile latest message so the next call stays within the lookback window", async () => {
+    // Reproduces the memory-retrospective / heartbeat case: a long run of
+    // autonomous assistant + tool turns sits between the previous user message
+    // and the volatile latest one. The 1h anchor moves back onto the previous
+    // user message (now >20 content blocks away); without a breakpoint on the
+    // latest message the next request's anchor would land beyond Anthropic's
+    // ~20-block lookback and re-create the whole prefix. The 5m tail bridge
+    // gives the next call an exact, reachable boundary.
+    const tail: Message[] = [];
+    for (let i = 0; i < 8; i++) {
+      tail.push(
+        assistantMsg(`heartbeat ${i}`),
+        toolUseMsg(`hb_${i}`, "bash"),
+        toolResultMsg(`hb_${i}`, "wake"),
+      );
+    }
+    // End the tail on an assistant message so the latest user message is not
+    // merged with a preceding tool_result (both user role).
+    tail.push(assistantMsg("still here"));
+    const messages: Message[] = [
+      userMsg("Turn 1"),
+      ...tail,
+      userMsg("Turn 2 (volatile)"),
+    ];
+    await provider.sendMessage(messages, {
+      config: { mutableLatestUserMessage: true },
+    });
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        text?: string;
+        cache_control?: { type: string; ttl?: string };
+      }>;
+    }>;
+    const findUser = (text: string) =>
+      sent.find(
+        (m) => m.role === "user" && m.content.some((b) => b.text === text),
+      )!;
+
+    // The volatile latest message gets the 5m tail bridge.
+    const turn2 = findUser("Turn 2 (volatile)");
+    const turn2Last = turn2.content[turn2.content.length - 1];
+    expect(turn2Last.cache_control).toEqual({ type: "ephemeral", ttl: "5m" });
+
+    // The previous stable user message — across the whole autonomous tail —
+    // is the 1h anchor.
+    const turn1 = findUser("Turn 1");
+    const turn1Last = turn1.content[turn1.content.length - 1];
+    expect(turn1Last.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
+  });
+
+  test("mutableLatestUserMessage absent: the same long autonomous tail gets no extra tail bridge (placement unchanged)", async () => {
+    // Regression guard: the tail bridge is gated on the volatile-latest flag,
+    // not on tail length. With the flag off, the latest message keeps the
+    // ordinary 1h turn-start anchor and no 5m bridge is added.
+    const tail: Message[] = [];
+    for (let i = 0; i < 8; i++) {
+      tail.push(
+        assistantMsg(`heartbeat ${i}`),
+        toolUseMsg(`hb_${i}`, "bash"),
+        toolResultMsg(`hb_${i}`, "wake"),
+      );
+    }
+    // End the tail on an assistant message so the latest user message is not
+    // merged with a preceding tool_result (both user role).
+    tail.push(assistantMsg("still here"));
+    const messages: Message[] = [userMsg("Turn 1"), ...tail, userMsg("Turn 2")];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        text?: string;
+        cache_control?: { type: string; ttl?: string };
+      }>;
+    }>;
+    const findUser = (text: string) =>
+      sent.find(
+        (m) => m.role === "user" && m.content.some((b) => b.text === text),
+      )!;
+
+    // Latest message keeps the 1h turn-start anchor — no 5m bridge.
+    const turn2 = findUser("Turn 2");
+    const turn2Last = turn2.content[turn2.content.length - 1];
+    expect(turn2Last.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
+    // Previous-turn anchor keeps 1h.
+    const turn1 = findUser("Turn 1");
+    const turn1Last = turn1.content[turn1.content.length - 1];
+    expect(turn1Last.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
+  });
+
+  test("mutableLatestUserMessage + disableTurnStartCache: the first-of-turn tail bridge is suppressed on the volatile latest block", async () => {
+    // The volatile-latest tail bridge lands on the turn-start block, so it must
+    // honor disableTurnStartCache (the explicit opt-out from paid cache writes
+    // on volatile turn-start content) just like the long-TTL anchor does. The
+    // previous-turn anchor is unaffected — it targets a stable prior block.
+    const messages: Message[] = [
+      userMsg("Turn 1"),
+      assistantMsg("Response 1"),
+      userMsg("Turn 2 (volatile)"),
+    ];
+    await provider.sendMessage(messages, {
+      config: { mutableLatestUserMessage: true, disableTurnStartCache: true },
+    });
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        text: string;
+        cache_control?: { type: string; ttl?: string };
+      }>;
+    }>;
+    const userMessages = sent.filter((m) => m.role === "user");
+
+    // Latest (turn-start) user message gets NO breakpoint — disableTurnStartCache
+    // suppresses the 5m bridge.
+    const latest = userMessages[userMessages.length - 1];
+    for (const block of latest.content) {
+      expect(block.cache_control).toBeUndefined();
+    }
+
+    // Previous stable user message (Turn 1) still gets the 1h anchor.
+    const prev = userMessages[userMessages.length - 2];
+    const prevLast = prev.content[prev.content.length - 1];
+    expect(prevLast.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
+  });
+
   test("mutableLatestUserMessage is not forwarded to the Anthropic API", async () => {
     await provider.sendMessage([userMsg("Hi")], {
       config: { mutableLatestUserMessage: true },
@@ -3057,3 +3191,70 @@ describe("AnthropicProvider — thinking block send-time filtering", () => {
     expect(signatures).toContain("sig-step2");
   });
 });
+
+describe("AnthropicProvider — deprecated sampling params (temperature / top_p / top_k)", () => {
+  beforeEach(() => {
+    lastStreamParams = null;
+  });
+
+  // opus-4-7 / opus-4-8 (and, conservatively, fable) reject `temperature`,
+  // `top_p`, and `top_k` with a 400; the provider must strip all three.
+  for (const model of [
+    "claude-opus-4-8",
+    "claude-opus-4-7",
+    "claude-fable-5",
+  ]) {
+    test(`strips temperature, top_p, and top_k for ${model}`, async () => {
+      const provider = new AnthropicProvider("sk-ant-test", model);
+      await provider.sendMessage([userMsg("Hi")], {
+        systemPrompt: "You are helpful.",
+        config: { temperature: 0, top_p: 0.95, top_k: 40 },
+      });
+      expect(lastStreamParams!).not.toHaveProperty("temperature");
+      expect(lastStreamParams!).not.toHaveProperty("top_p");
+      expect(lastStreamParams!).not.toHaveProperty("top_k");
+    });
+  }
+
+  // opus-4-6 / sonnet-4-6 still accept the params — they must pass through,
+  // including `temperature: 0` (a value check, not truthiness).
+  test("forwards temperature (including 0), top_p, and top_k for opus-4-6", async () => {
+    const provider = new AnthropicProvider("sk-ant-test", "claude-opus-4-6");
+    await provider.sendMessage([userMsg("Hi")], {
+      systemPrompt: "You are helpful.",
+      config: { temperature: 0, top_p: 0.95, top_k: 40 },
+    });
+    expect(lastStreamParams!.temperature).toBe(0);
+    expect(lastStreamParams!.top_p).toBe(0.95);
+    expect(lastStreamParams!.top_k).toBe(40);
+  });
+
+  test("forwards temperature, top_p, and top_k for sonnet-4-6", async () => {
+    const provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
+    await provider.sendMessage([userMsg("Hi")], {
+      systemPrompt: "You are helpful.",
+      config: { temperature: 0.7, top_p: 0.9, top_k: 20 },
+    });
+    expect(lastStreamParams!.temperature).toBe(0.7);
+    expect(lastStreamParams!.top_p).toBe(0.9);
+    expect(lastStreamParams!.top_k).toBe(20);
+  });
+
+  // A per-call model override targeting a deprecating model must win over the
+  // provider's default (accepting) model.
+  test("strips params when a per-call model override deprecates them", async () => {
+    const provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
+    await provider.sendMessage([userMsg("Hi")], {
+      systemPrompt: "You are helpful.",
+      config: {
+        temperature: 0,
+        top_p: 0.95,
+        top_k: 40,
+        model: "claude-opus-4-8",
+      },
+    });
+    expect(lastStreamParams!).not.toHaveProperty("temperature");
+    expect(lastStreamParams!).not.toHaveProperty("top_p");
+    expect(lastStreamParams!).not.toHaveProperty("top_k");
+  });
+});

package/src/__tests__/app-builder-skill-instructions.test.ts

@@ -1,12 +1,24 @@
-import { readFile } from "node:fs/promises";
+import { readdir, readFile } from "node:fs/promises";
 import { describe, expect, test } from "bun:test";
 
+const skillDir = new URL(
+  "../config/bundled-skills/app-builder/",
+  import.meta.url,
+);
+
+async function readAllSkillMarkdown(): Promise<string> {
+  const entries = await readdir(skillDir, { recursive: true });
+  const contents = await Promise.all(
+    entries
+      .filter((name) => name.endsWith(".md"))
+      .map((name) => readFile(new URL(name, skillDir), "utf8")),
+  );
+  return contents.join("\n");
+}
+
 describe("app-builder skill instructions", () => {
   test("uses non-CLI UI confirmation for optional profile switch", async () => {
-    const skillText = await readFile(
-      new URL("../config/bundled-skills/app-builder/SKILL.md", import.meta.url),
-      "utf8",
-    );
+    const skillText = await readFile(new URL("SKILL.md", skillDir), "utf8");
     const preflightStart = skillText.indexOf("### 0. Preflight");
     const preflightEnd = skillText.indexOf("### 1. Gather Requirements");
     const preflight = skillText.slice(preflightStart, preflightEnd);
@@ -19,4 +31,34 @@ describe("app-builder skill instructions", () => {
       "Do not call the shell command `assistant ui confirm`",
     );
   });
+
+  test("does not reference window.vellum.* APIs the sandbox no longer injects", async () => {
+    const skillText = await readAllSkillMarkdown();
+    // These were provided by the retired Swift macOS runtime; the current
+    // web/Electron/Capacitor sandbox never injects them, so the skill must not
+    // steer generated apps toward APIs that silently no-op at runtime.
+    for (const removedApi of [
+      "vellum.widgets",
+      "vellum.confirm",
+      "vellum.openLink",
+      "vellum.theme",
+      "vellum-theme-change",
+      "vellum.data",
+    ]) {
+      expect(skillText).not.toContain(removedApi);
+    }
+  });
+
+  test("documents the window.vellum.* APIs the sandbox actually provides", async () => {
+    const skillText = await readAllSkillMarkdown();
+    for (const realApi of [
+      "window.vellum.sendAction",
+      "window.vellum.fetch",
+      "window.vellum.route",
+      "relay_prompt",
+      "set_view",
+    ]) {
+      expect(skillText).toContain(realApi);
+    }
+  });
 });

package/src/__tests__/app-conversation-ids-backfill.test.ts

@@ -21,7 +21,7 @@ import { initializeDb } from "../memory/db-init.js";
 import { rawRun } from "../memory/raw-query.js";
 
 // Initialize db once for all tests
-initializeDb();
+await initializeDb();
 
 // ---------------------------------------------------------------------------
 // Helpers