@vellumai/assistant 0.11.2 → 0.11.3-staging.2

package/src/__tests__/subagent-role-resolution.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, test } from "bun:test";
+
+import { resolveSubagentRole } from "../subagent/role-resolution.js";
+
+describe("resolveSubagentRole", () => {
+  test("an absent role runs as builder", () => {
+    // A spawn that names no role has always been write-capable.
+    expect(resolveSubagentRole(undefined)).toEqual({ role: "builder" });
+  });
+
+  test("a blank role is the same as naming none", () => {
+    for (const raw of ["", " ", "\n\t "]) {
+      expect(resolveSubagentRole(raw)).toEqual({ role: "builder" });
+    }
+  });
+
+  test("the three type names pass through unchanged", () => {
+    expect(resolveSubagentRole("researcher")).toEqual({ role: "researcher" });
+    expect(resolveSubagentRole("builder")).toEqual({ role: "builder" });
+    expect(resolveSubagentRole("advisor")).toEqual({ role: "advisor" });
+  });
+
+  test("type names are case- and whitespace-insensitive", () => {
+    expect(resolveSubagentRole("  Researcher ")).toEqual({
+      role: "researcher",
+    });
+    expect(resolveSubagentRole("ADVISOR")).toEqual({ role: "advisor" });
+  });
+
+  test("legacy read-only names resolve to researcher and report the alias", () => {
+    expect(resolveSubagentRole("planner")).toEqual({
+      role: "researcher",
+      alias: "planner",
+    });
+    expect(resolveSubagentRole("Investigator")).toEqual({
+      role: "researcher",
+      alias: "investigator",
+    });
+  });
+
+  test("legacy write-capable names resolve to builder and report the alias", () => {
+    expect(resolveSubagentRole("coder")).toEqual({
+      role: "builder",
+      alias: "coder",
+    });
+    expect(resolveSubagentRole("general")).toEqual({
+      role: "builder",
+      alias: "general",
+    });
+  });
+
+  test("an unrecognized role runs read-only with the text as persona", () => {
+    // Least privilege: an invented or misspelled role must never hand out
+    // write access.
+    expect(resolveSubagentRole("financial journalist")).toEqual({
+      role: "researcher",
+      personaText: "financial journalist",
+    });
+    expect(resolveSubagentRole("buildr")).toEqual({
+      role: "researcher",
+      personaText: "buildr",
+    });
+  });
+
+  test("persona text is collapsed to a single line", () => {
+    const resolved = resolveSubagentRole("  staff\n  security   engineer  ");
+    expect(resolved.role).toBe("researcher");
+    expect(resolved.personaText).toBe("staff security engineer");
+  });
+
+  test("a sentence-length role is truncated to a persona-sized hint", () => {
+    const sentence =
+      "You are a meticulous senior staff engineer who reviews every change against the design document and reports every discrepancy you can find, however small.";
+    const resolved = resolveSubagentRole(sentence);
+    expect(resolved.role).toBe("researcher");
+    expect(resolved.personaText!.length).toBeLessThanOrEqual(124);
+    expect(resolved.personaText!.endsWith("...")).toBe(true);
+    expect(resolved.personaText!.startsWith("You are a meticulous")).toBe(true);
+  });
+
+  test("inherited object members are not roles", () => {
+    for (const raw of ["constructor", "toString", "__proto__"]) {
+      const resolved = resolveSubagentRole(raw);
+      expect(resolved.role).toBe("researcher");
+      expect(resolved.personaText).toBe(raw);
+    }
+  });
+});

package/src/__tests__/subagent-spawn-and-await.test.ts

@@ -68,6 +68,11 @@ class FakeConversation {
   messages: Message[];
   usageStats = { inputTokens: 10, outputTokens: 5, estimatedCost: 0.001 };
   subagentDeniedToolNames = new Set<string>();
+  subagentToolStats = {
+    calls: 0,
+    succeeded: 0,
+    filesWritten: new Set<string>(),
+  };
   conversationType = "background";
   hasSystemPromptOverride = false;
 
@@ -394,6 +399,49 @@ describe("SubagentManager.spawnAndAwait", () => {
     expect(chunks).toEqual(["Hello ", "(pondering) ", "world"]);
   });
 
+  test("reports tool activity via onProgress, which onText never sees", async () => {
+    // A subagent executing a tool streams no delta, so a caller bounding the
+    // run by an idle window has no signal from onText alone. onProgress is that
+    // signal, and it is a strict superset: text deltas count as progress too.
+    nextConversationConfig = {
+      messages: [
+        { role: "assistant", content: [{ type: "text", text: "done" }] },
+      ],
+      emitDeltas: [
+        { type: "assistant_text_delta", text: "Reading " } as AssistantEvent,
+        {
+          type: "tool_use_start",
+          toolName: "file_read",
+          input: { path: "a.ts" },
+        } as unknown as AssistantEvent,
+        {
+          type: "tool_output_chunk",
+          chunk: "export const a = 1;",
+        } as unknown as AssistantEvent,
+        {
+          type: "tool_result",
+          result: "export const a = 1;",
+        } as unknown as AssistantEvent,
+        // Lifecycle chatter is not progress.
+        { type: "subagent_status_changed" } as AssistantEvent,
+      ],
+    };
+
+    const chunks: string[] = [];
+    let progressCount = 0;
+    const manager = new SubagentManager();
+    await manager.spawnAndAwait(makeConfig(), () => {}, {
+      onText: (chunk) => chunks.push(chunk),
+      onProgress: () => {
+        progressCount++;
+      },
+    });
+
+    // One text delta plus three tool events; the status event is excluded.
+    expect(progressCount).toBe(4);
+    expect(chunks).toEqual(["Reading "]);
+  });
+
   test("aborting the provided signal rejects the run", async () => {
     nextConversationConfig = { waitForAbort: true };
 

package/src/__tests__/subagent-spawn-tool-fork.test.ts

@@ -37,6 +37,7 @@ import {
 } from "../daemon/conversation-registry.js";
 import type { Message } from "../providers/types.js";
 import { getSubagentManager } from "../subagent/index.js";
+import { SUBAGENT_ROLE_REGISTRY } from "../subagent/types.js";
 import { executeSubagentSpawn } from "../tools/subagent/spawn.js";
 
 // ── Shared helpers ──────────────────────────────────────────────────
@@ -113,18 +114,61 @@ describe("subagent_spawn fork parameter", () => {
     }
   });
 
-  test("fork: true ignores role parameter", async () => {
+  test.each(["researcher", "builder"])(
+    "fork: true honors role %s",
+    async (role) => {
+      const manager = getSubagentManager();
+      const originalSpawn = manager.spawn.bind(manager);
+
+      let capturedConfig: Record<string, unknown> | undefined;
+      manager.spawn = async (config: Record<string, unknown>) => {
+        capturedConfig = config;
+        return "fork-role-id";
+      };
+
+      clearConversations();
+      setConversation("parent-conv-role", {
+        messages: FAKE_PARENT_MESSAGES,
+        getCurrentSystemPrompt: () => "Parent prompt.",
+      } as any);
+
+      try {
+        const result = await executeSubagentSpawn(
+          {
+            label: "Fork with role",
+            objective: "Do something",
+            fork: true,
+            role,
+          },
+          makeContext("parent-conv-role", { sendToClient: () => {} }),
+        );
+
+        expect(result.isError).toBe(false);
+        expect(capturedConfig).toBeDefined();
+        // The manager applies the type's allowlist to a fork like any other
+        // spawn, so a read-only fork really is read-only.
+        expect(capturedConfig!.role).toBe(role);
+        expect(capturedConfig!.fork).toBe(true);
+        expect(JSON.parse(result.content).role).toBe(role);
+      } finally {
+        manager.spawn = originalSpawn;
+        clearConversations();
+      }
+    },
+  );
+
+  test("a fork that names no role keeps the parent's tool surface", async () => {
     const manager = getSubagentManager();
     const originalSpawn = manager.spawn.bind(manager);
 
     let capturedConfig: Record<string, unknown> | undefined;
     manager.spawn = async (config: Record<string, unknown>) => {
       capturedConfig = config;
-      return "fork-role-id";
+      return "fork-no-role-id";
     };
 
     clearConversations();
-    setConversation("parent-conv-role", {
+    setConversation("parent-conv-no-role", {
       messages: FAKE_PARENT_MESSAGES,
       getCurrentSystemPrompt: () => "Parent prompt.",
     } as any);
@@ -132,19 +176,19 @@ describe("subagent_spawn fork parameter", () => {
     try {
       const result = await executeSubagentSpawn(
         {
-          label: "Fork with role",
+          label: "Plain fork",
           objective: "Do something",
           fork: true,
-          role: "researcher", // should be ignored
         },
-        makeContext("parent-conv-role", { sendToClient: () => {} }),
+        makeContext("parent-conv-no-role", { sendToClient: () => {} }),
       );
 
       expect(result.isError).toBe(false);
-      expect(capturedConfig).toBeDefined();
-      // When fork is true, role should NOT be passed to the manager config
-      expect(capturedConfig!.role).toBeUndefined();
-      expect(capturedConfig!.fork).toBe(true);
+      // No role resolves to builder, which declares no allowlist, so the fork
+      // keeps the full surface described by the system prompt it inherits.
+      expect(capturedConfig!.role).toBe("builder");
+      expect(SUBAGENT_ROLE_REGISTRY.builder.allowedTools).toBeUndefined();
+      expect(JSON.parse(result.content).role).toBe("builder");
     } finally {
       manager.spawn = originalSpawn;
       clearConversations();
@@ -301,6 +345,63 @@ describe("subagent_spawn fork parameter", () => {
     }
   });
 
+  // Spawn mode is what makes delegated LLM spend separable: every variety
+  // emits under `llm_call_site = "subagentSpawn"`, so the mode declared here
+  // is the only thing distinguishing a fresh spawn from a context-inheriting
+  // fork downstream.
+  test("declares spawnMode 'fork' for a forked spawn", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+
+    let capturedConfig: Record<string, unknown> | undefined;
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "fork-mode-id";
+    };
+
+    clearConversations();
+    setConversation("spawn-mode-parent", {
+      messages: FAKE_PARENT_MESSAGES,
+      getCurrentSystemPrompt: () => "You are a helpful assistant.",
+    } as any);
+
+    try {
+      await executeSubagentSpawn(
+        { label: "Forked", objective: "Continue", fork: true },
+        makeContext("spawn-mode-parent", { sendToClient: () => {} }),
+      );
+
+      expect(capturedConfig!.spawnMode).toBe("fork");
+    } finally {
+      manager.spawn = originalSpawn;
+      clearConversations();
+    }
+  });
+
+  test("declares spawnMode 'regular' for a non-forked spawn", async () => {
+    const manager = getSubagentManager();
+    const originalSpawn = manager.spawn.bind(manager);
+
+    let capturedConfig: Record<string, unknown> | undefined;
+    manager.spawn = async (config: Record<string, unknown>) => {
+      capturedConfig = config;
+      return "regular-mode-id";
+    };
+
+    try {
+      await executeSubagentSpawn(
+        { label: "Plain", objective: "Do work", role: "researcher" },
+        makeContext("spawn-mode-conv", { sendToClient: () => {} }),
+      );
+
+      expect(capturedConfig!.spawnMode).toBe("regular");
+      // Role and spawn mode are orthogonal dimensions, both are recorded.
+      expect(capturedConfig!.role).toBe("researcher");
+    } finally {
+      manager.spawn = originalSpawn;
+    }
+  });
+
   test("error when parent conversation cannot be resolved", async () => {
     // Empty store — findConversation will return undefined.
     clearConversations();

package/src/__tests__/subagent-terminal-message.test.ts

@@ -112,7 +112,7 @@ describe("buildSubagentTerminalMessage", () => {
     expect(msg).toContain("attempted file_write");
     expect(msg).toContain("does not permit it");
     expect(msg).toContain("re-spawn with a role that includes it");
-    expect(msg).toContain("coder");
+    expect(msg).toContain("builder");
   });
 
   test("appends a pluralized denied-tools note to a read-pointer completion", () => {
@@ -146,4 +146,91 @@ describe("buildSubagentTerminalMessage", () => {
     expect(msg).not.toContain("does not permit");
     expect(msg).not.toContain("re-spawn");
   });
+
+  test("appends the machine stats footer to an inlined completion", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "refactor",
+      subagentId: "sa-10",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "Renamed the helper across the module.",
+      stats: { calls: 7, succeeded: 6, filesWritten: 2 },
+    });
+
+    expect(msg).toContain("Renamed the helper across the module.");
+    expect(msg).toContain(
+      "[stats: 7 tool calls, 6 succeeded, files written via file_write/file_edit: 2]",
+    );
+  });
+
+  test("appends the machine stats footer to a read-pointer completion", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "quiet",
+      subagentId: "sa-11",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "   ",
+      stats: { calls: 1, succeeded: 1, filesWritten: 0 },
+    });
+
+    expect(msg).toContain('subagent_read with subagent_id "sa-11"');
+    // Singular reading for a single call, so the footer never says "1 tool calls".
+    expect(msg).toContain(
+      "[stats: 1 tool call, 1 succeeded, files written via file_write/file_edit: 0]",
+    );
+  });
+
+  test("a zero-call run gets the unverified warning instead of a count", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "claims-work",
+      subagentId: "sa-12",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "I created the report and saved it to disk.",
+      stats: { calls: 0, succeeded: 0, filesWritten: 0 },
+    });
+
+    expect(msg).toContain(
+      "[stats: no tools were used by this subagent; treat any claims of executed work as unverified]",
+    );
+    expect(msg).not.toContain("tool calls,");
+  });
+
+  test("the file count names the two tools it comes from", () => {
+    // A builder runs on the parent's whole tool surface and can write with a
+    // bash heredoc, a document tool, or an MCP tool, none of which the counter
+    // sees. A bare "files written: 0" next to a builder reporting the files it
+    // created would read as fabricated work, so the footer says which
+    // instrument it measured.
+    const msg = buildSubagentTerminalMessage({
+      label: "shell-builder",
+      subagentId: "sa-14",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "Created scripts/export.py with a heredoc.",
+      stats: { calls: 14, succeeded: 14, filesWritten: 0 },
+    });
+
+    expect(msg).toContain(
+      "[stats: 14 tool calls, 14 succeeded, files written via file_write/file_edit: 0]",
+    );
+    expect(msg).not.toContain("succeeded, files written: 0");
+  });
+
+  test("omits the stats footer when no stats were harvested", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "no-stats",
+      subagentId: "sa-13",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "Done.",
+    });
+
+    expect(msg).not.toContain("[stats:");
+  });
 });