@vellumai/assistant 0.9.0 → 0.10.0-staging.2

package/src/__tests__/subagent-tools.test.ts

@@ -6,6 +6,28 @@ import { describe, expect, mock, test } from "bun:test";
 let mockGetMessages: (
   conversationId: string,
 ) => Array<{ role: string; content: string }> | null = () => null;
+const mockProfiles = {
+  balanced: {},
+  "cost-optimized": {},
+  disabled: { status: "disabled" },
+  "quality-optimized": {},
+};
+mock.module("../config/loader.js", () => ({
+  getConfigReadOnly: () => ({
+    llm: { profiles: mockProfiles },
+  }),
+  getConfig: () => ({
+    llm: {
+      default: {
+        provider: "anthropic",
+        provider_connection: "anthropic-managed",
+        model: "claude-opus-4-7",
+      },
+      profiles: mockProfiles,
+    },
+    rateLimit: { maxRequestsPerMinute: 0 },
+  }),
+}));
 mock.module("../memory/conversation-crud.js", () => ({
   setConversationOriginChannelIfUnset: () => {},
   updateConversationContextWindow: () => {},
@@ -140,6 +162,7 @@ describe("Subagent tool definitions", () => {
     const def = findTool("subagent_spawn");
     expect(def).toBeDefined();
     expect(def.input_schema.required).toEqual(["label", "objective"]);
+    expect(def.input_schema.properties.inference_profile).toBeDefined();
   });
 
   test("abort tool has correct definition", () => {
@@ -422,6 +445,249 @@ describe("Subagent spawn success and failure", () => {
     }
   });
 
+  test("spawn passes explicit inference_profile to manager over inherited override", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let capturedConfig: Record<string, unknown> | undefined;
+
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "profile-subagent-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        {
+          label: "Profile test",
+          objective: "Do it with a chosen model profile",
+          inference_profile: "quality-optimized",
+        },
+        makeContext("sess-spawn-profile", {
+          sendToClient: () => {},
+          overrideProfile: "balanced",
+        }),
+      );
+
+      expect(result.isError).toBe(false);
+      expect(capturedConfig).toBeDefined();
+      expect(capturedConfig!.overrideProfile).toBe("quality-optimized");
+      expect(capturedConfig!.forceOverrideProfile).toBe(true);
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn inherits the invoking call site's default profile when no override is present", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let capturedConfig: Record<string, unknown> | undefined;
+
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "inherit-default-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        { label: "Inherit default", objective: "Do it" },
+        makeContext("sess-inherit-default", {
+          sendToClient: () => {},
+          invokingCallSite: "mainAgent",
+        }),
+      );
+
+      expect(result.isError).toBe(false);
+      // No explicit profile and no per-turn override → the child matches the
+      // invoking call site's resolved default profile (balanced for mainAgent
+      // in the test config).
+      expect(capturedConfig!.overrideProfile).toBe("balanced");
+      expect(capturedConfig!.forceOverrideProfile).toBeUndefined();
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn inherits a non-main invoker's call-site default profile", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let capturedConfig: Record<string, unknown> | undefined;
+
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "inherit-heartbeat-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        { label: "Heartbeat child", objective: "Do it" },
+        makeContext("sess-inherit-heartbeat", {
+          sendToClient: () => {},
+          invokingCallSite: "heartbeatAgent",
+        }),
+      );
+
+      expect(result.isError).toBe(false);
+      // A subagent spawned from a heartbeat turn matches heartbeatAgent's own
+      // cost-optimized default, not the mainAgent default.
+      expect(capturedConfig!.overrideProfile).toBe("cost-optimized");
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn prefers a per-turn override profile over the invoker default", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let capturedConfig: Record<string, unknown> | undefined;
+
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "inherit-override-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        { label: "Override child", objective: "Do it" },
+        makeContext("sess-inherit-override", {
+          sendToClient: () => {},
+          invokingCallSite: "mainAgent",
+          overrideProfile: "quality-optimized",
+        }),
+      );
+
+      expect(result.isError).toBe(false);
+      // The live per-turn override (per-conversation or tool-routed) wins over
+      // the call-site default, and is forwarded non-forced.
+      expect(capturedConfig!.overrideProfile).toBe("quality-optimized");
+      expect(capturedConfig!.forceOverrideProfile).toBeUndefined();
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn skips the auto profile so the child keeps its own default", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let capturedConfig: Record<string, unknown> | undefined;
+
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "inherit-auto-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        { label: "Auto child", objective: "Do it" },
+        makeContext("sess-inherit-auto", {
+          sendToClient: () => {},
+          invokingCallSite: "mainAgent",
+          // "auto" is metadata-only; forwarding it would collapse the child to
+          // llm.default, so the inherited path drops it and the child keeps its
+          // own subagentSpawn default.
+          overrideProfile: "auto",
+        }),
+      );
+
+      expect(result.isError).toBe(false);
+      expect(capturedConfig!.overrideProfile).toBeUndefined();
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn still forces an explicit inference_profile over the invoker default", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let capturedConfig: Record<string, unknown> | undefined;
+
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "inherit-explicit-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        {
+          label: "Explicit child",
+          objective: "Do it",
+          inference_profile: "cost-optimized",
+        },
+        makeContext("sess-inherit-explicit", {
+          sendToClient: () => {},
+          invokingCallSite: "mainAgent",
+        }),
+      );
+
+      expect(result.isError).toBe(false);
+      expect(capturedConfig!.overrideProfile).toBe("cost-optimized");
+      expect(capturedConfig!.forceOverrideProfile).toBe(true);
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn returns error for unknown inference_profile", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let spawnCalled = false;
+
+    manager.spawn = async () => {
+      spawnCalled = true;
+      return "profile-subagent-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        {
+          label: "Bad profile",
+          objective: "Do it",
+          inference_profile: "does-not-exist",
+        },
+        makeContext("sess-spawn-bad-profile", { sendToClient: () => {} }),
+      );
+
+      expect(result.isError).toBe(true);
+      expect(result.content).toContain(
+        'Inference profile "does-not-exist" is not defined',
+      );
+      expect(spawnCalled).toBe(false);
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
+  test("spawn returns error for disabled inference_profile", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+    let spawnCalled = false;
+
+    manager.spawn = async () => {
+      spawnCalled = true;
+      return "profile-subagent-id";
+    };
+
+    try {
+      const result = await executeSubagentSpawn(
+        {
+          label: "Disabled profile",
+          objective: "Do it",
+          inference_profile: "disabled",
+        },
+        makeContext("sess-spawn-disabled-profile", {
+          sendToClient: () => {},
+        }),
+      );
+
+      expect(result.isError).toBe(true);
+      expect(result.content).toContain(
+        'Inference profile "disabled" is disabled',
+      );
+      expect(spawnCalled).toBe(false);
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
   test("spawn handles non-Error throws gracefully", async () => {
     const manager = getSubagentManager();
     const originalSpawn = manager.spawn.bind(manager);

package/src/__tests__/surface-completion-nudge-hook.test.ts

@@ -0,0 +1,367 @@
+/**
+ * Tests for the default `surface-completion-nudge` plugin's hooks.
+ *
+ * Covers:
+ * - The `post-model-call` hook nudges (continue + canonical text appended as a
+ *   `user` message) when a turn ends with a progress surface left open: a
+ *   `task_progress` card shown and never advanced to a terminal status, and a
+ *   `work_result` shown `in_progress`.
+ * - It does NOT nudge when the surface was completed via `ui_update`, dismissed
+ *   via `ui_dismiss`, or was never a progress surface (a plain card / a form).
+ * - Outcomes it does not own are ignored: a provider rejection, a tool-bearing
+ *   turn, and a non-main-agent call site.
+ * - The signal is scoped to the current response cycle — a surface left open in
+ *   a prior cycle (before the last genuine user prompt) does not trigger it.
+ * - The one-shot bound is split across the two hooks: `post-model-call` marks it
+ *   (nudging at most once per run) and `stop` clears it so the next run nudges
+ *   afresh.
+ *
+ * The loop's actual continuation side-effects live in `agent/loop.ts` and are
+ * covered by integration tests. This file isolates the hook.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type {
+  PluginLogger,
+  PostModelCallContext,
+} from "../plugin-api/types.js";
+import postModelCall, {
+  SURFACE_COMPLETION_NUDGE_TEXT,
+} from "../plugins/defaults/surface-completion-nudge/hooks/post-model-call.js";
+import stop from "../plugins/defaults/surface-completion-nudge/hooks/stop.js";
+import {
+  isSurfaceCompletionNudged,
+  resetSurfaceCompletionNudgeStoreForTests,
+} from "../plugins/defaults/surface-completion-nudge/nudge-state-store.js";
+import type { ContentBlock, Message } from "../providers/types.js";
+
+// ─── Fixtures ────────────────────────────────────────────────────────────────
+
+const noopLogger: PluginLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+
+const finalText: ContentBlock = { type: "text", text: "All set." };
+
+let surfaceCounter = 0;
+
+/**
+ * An assistant `ui_show` turn paired with its `{ surfaceId }` tool result.
+ * Returns both messages plus the assigned surface id.
+ */
+function showSurface(input: Record<string, unknown>): {
+  messages: Message[];
+  surfaceId: string;
+} {
+  surfaceCounter += 1;
+  const toolUseId = `tu_show_${surfaceCounter}`;
+  const surfaceId = `surface-${surfaceCounter}`;
+  return {
+    surfaceId,
+    messages: [
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: toolUseId, name: "ui_show", input }],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: toolUseId,
+            content: JSON.stringify({ surfaceId }),
+          },
+        ],
+      },
+    ],
+  };
+}
+
+function updateSurface(
+  surfaceId: string,
+  data: Record<string, unknown>,
+): Message {
+  return {
+    role: "assistant",
+    content: [
+      {
+        type: "tool_use",
+        id: `tu_update_${surfaceId}`,
+        name: "ui_update",
+        input: { surface_id: surfaceId, data },
+      },
+    ],
+  };
+}
+
+function dismissSurface(surfaceId: string): Message {
+  return {
+    role: "assistant",
+    content: [
+      {
+        type: "tool_use",
+        id: `tu_dismiss_${surfaceId}`,
+        name: "ui_dismiss",
+        input: { surface_id: surfaceId },
+      },
+    ],
+  };
+}
+
+function userPrompt(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+const taskProgressShow = (status?: string): Record<string, unknown> => ({
+  surface_type: "card",
+  title: "Working on X",
+  data: {
+    template: "task_progress",
+    templateData: {
+      title: "Working on X",
+      ...(status ? { status } : {}),
+      steps: [{ label: "Step 1", status: status ?? "in_progress" }],
+    },
+  },
+});
+
+function makeCtx(
+  overrides: Partial<PostModelCallContext> = {},
+): PostModelCallContext {
+  return {
+    conversationId: "conv-scn",
+    callSite: "mainAgent",
+    content: [finalText],
+    messages: [],
+    stopReason: null,
+    decision: "stop",
+    logger: noopLogger,
+    ...overrides,
+  };
+}
+
+beforeEach(() => {
+  resetSurfaceCompletionNudgeStoreForTests();
+  surfaceCounter = 0;
+});
+
+// ─── Nudges when a progress surface is left open ──────────────────────────────
+
+describe("surface-completion-nudge — nudges on a dangling progress surface", () => {
+  test("task_progress card shown and never completed → continue with nudge", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("do the thing"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("continue");
+    const last = ctx.messages[ctx.messages.length - 1];
+    expect(last.role).toBe("user");
+    expect(last.content[0]).toEqual({
+      type: "text",
+      text: SURFACE_COMPLETION_NUDGE_TEXT,
+    });
+    expect(isSurfaceCompletionNudged("conv-scn")).toBe(true);
+  });
+
+  test("task_progress card shown with no explicit status → continue with nudge", async () => {
+    const shown = showSurface(taskProgressShow());
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("continue");
+  });
+
+  test("work_result shown in_progress → continue with nudge", async () => {
+    const shown = showSurface({
+      surface_type: "work_result",
+      data: { status: "in_progress", summary: "Crunching" },
+    });
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("continue");
+  });
+});
+
+// ─── Stays quiet when there is nothing to close ───────────────────────────────
+
+describe("surface-completion-nudge — no nudge when surface is closed or absent", () => {
+  test("task_progress completed via ui_update → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [
+        userPrompt("go"),
+        ...shown.messages,
+        updateSurface(shown.surfaceId, {
+          templateData: { status: "completed" },
+        }),
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+    expect(isSurfaceCompletionNudged("conv-scn")).toBe(false);
+  });
+
+  test("progress surface dismissed via ui_dismiss → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [
+        userPrompt("go"),
+        ...shown.messages,
+        dismissSurface(shown.surfaceId),
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("work_result shown completed → stop", async () => {
+    const shown = showSurface({
+      surface_type: "work_result",
+      data: { status: "completed", summary: "Done" },
+    });
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("plain card (not a progress surface) → stop", async () => {
+    const shown = showSurface({
+      surface_type: "card",
+      title: "Weather",
+      data: { template: "weather_forecast", body: "Sunny" },
+    });
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("no surfaces shown at all → stop", async () => {
+    const ctx = makeCtx({
+      messages: [userPrompt("go")],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+});
+
+// ─── Outcomes the hook does not own ───────────────────────────────────────────
+
+describe("surface-completion-nudge — ignores outcomes it does not own", () => {
+  test("provider rejection (error present) → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+      error: new Error("provider exploded"),
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("tool-bearing turn (model still working) → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+      content: [
+        { type: "tool_use", id: "tu_next", name: "read_file", input: {} },
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("non-main-agent call site → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+      callSite: "heartbeatAgent",
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+});
+
+// ─── Cycle scoping and the one-shot bound ─────────────────────────────────────
+
+describe("surface-completion-nudge — cycle scoping and one-shot bound", () => {
+  test("a surface left open in a prior cycle does not trigger this cycle", async () => {
+    const priorOpen = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [
+        userPrompt("first task"),
+        ...priorOpen.messages,
+        // New genuine user prompt opens a fresh cycle with no open surface.
+        userPrompt("second task"),
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("nudges at most once per run; stop clears the bound", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const firstCtx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(firstCtx);
+    expect(firstCtx.decision).toBe("continue");
+
+    // Same run, surface still open: the one-shot bound suppresses a second nudge.
+    const secondCtx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+    await postModelCall(secondCtx);
+    expect(secondCtx.decision).toBe("stop");
+
+    // Terminal stop clears the bound so the next run nudges afresh.
+    await stop({
+      conversationId: "conv-scn",
+      messages: [],
+      exitReason: "no_tool_calls",
+      logger: noopLogger,
+    });
+    expect(isSurfaceCompletionNudged("conv-scn")).toBe(false);
+
+    const thirdCtx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+    await postModelCall(thirdCtx);
+    expect(thirdCtx.decision).toBe("continue");
+  });
+});

package/src/__tests__/task-progress-nudge-hook.test.ts

@@ -104,7 +104,7 @@ function currentResponse(toolUseId: string): ToolResultContent {
   };
 }
 
-/** A weak-model id that matches WEAK_MODEL_PATTERN (the gated population). */
+/** A weak-model id that matches WEAK_OPEN_MODEL_PATTERN (the gated population). */
 const WEAK_MODEL = "minimax/minimax-m3";
 
 function makeCtx(