@vellumai/assistant 0.9.0 → 0.10.0-dev.202606192226.8ed8a6a

package/src/__tests__/surface-completion-nudge-hook.test.ts

@@ -0,0 +1,367 @@
+/**
+ * Tests for the default `surface-completion-nudge` plugin's hooks.
+ *
+ * Covers:
+ * - The `post-model-call` hook nudges (continue + canonical text appended as a
+ *   `user` message) when a turn ends with a progress surface left open: a
+ *   `task_progress` card shown and never advanced to a terminal status, and a
+ *   `work_result` shown `in_progress`.
+ * - It does NOT nudge when the surface was completed via `ui_update`, dismissed
+ *   via `ui_dismiss`, or was never a progress surface (a plain card / a form).
+ * - Outcomes it does not own are ignored: a provider rejection, a tool-bearing
+ *   turn, and a non-main-agent call site.
+ * - The signal is scoped to the current response cycle — a surface left open in
+ *   a prior cycle (before the last genuine user prompt) does not trigger it.
+ * - The one-shot bound is split across the two hooks: `post-model-call` marks it
+ *   (nudging at most once per run) and `stop` clears it so the next run nudges
+ *   afresh.
+ *
+ * The loop's actual continuation side-effects live in `agent/loop.ts` and are
+ * covered by integration tests. This file isolates the hook.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type {
+  PluginLogger,
+  PostModelCallContext,
+} from "../plugin-api/types.js";
+import postModelCall, {
+  SURFACE_COMPLETION_NUDGE_TEXT,
+} from "../plugins/defaults/surface-completion-nudge/hooks/post-model-call.js";
+import stop from "../plugins/defaults/surface-completion-nudge/hooks/stop.js";
+import {
+  isSurfaceCompletionNudged,
+  resetSurfaceCompletionNudgeStoreForTests,
+} from "../plugins/defaults/surface-completion-nudge/nudge-state-store.js";
+import type { ContentBlock, Message } from "../providers/types.js";
+
+// ─── Fixtures ────────────────────────────────────────────────────────────────
+
+const noopLogger: PluginLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+
+const finalText: ContentBlock = { type: "text", text: "All set." };
+
+let surfaceCounter = 0;
+
+/**
+ * An assistant `ui_show` turn paired with its `{ surfaceId }` tool result.
+ * Returns both messages plus the assigned surface id.
+ */
+function showSurface(input: Record<string, unknown>): {
+  messages: Message[];
+  surfaceId: string;
+} {
+  surfaceCounter += 1;
+  const toolUseId = `tu_show_${surfaceCounter}`;
+  const surfaceId = `surface-${surfaceCounter}`;
+  return {
+    surfaceId,
+    messages: [
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: toolUseId, name: "ui_show", input }],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: toolUseId,
+            content: JSON.stringify({ surfaceId }),
+          },
+        ],
+      },
+    ],
+  };
+}
+
+function updateSurface(
+  surfaceId: string,
+  data: Record<string, unknown>,
+): Message {
+  return {
+    role: "assistant",
+    content: [
+      {
+        type: "tool_use",
+        id: `tu_update_${surfaceId}`,
+        name: "ui_update",
+        input: { surface_id: surfaceId, data },
+      },
+    ],
+  };
+}
+
+function dismissSurface(surfaceId: string): Message {
+  return {
+    role: "assistant",
+    content: [
+      {
+        type: "tool_use",
+        id: `tu_dismiss_${surfaceId}`,
+        name: "ui_dismiss",
+        input: { surface_id: surfaceId },
+      },
+    ],
+  };
+}
+
+function userPrompt(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+const taskProgressShow = (status?: string): Record<string, unknown> => ({
+  surface_type: "card",
+  title: "Working on X",
+  data: {
+    template: "task_progress",
+    templateData: {
+      title: "Working on X",
+      ...(status ? { status } : {}),
+      steps: [{ label: "Step 1", status: status ?? "in_progress" }],
+    },
+  },
+});
+
+function makeCtx(
+  overrides: Partial<PostModelCallContext> = {},
+): PostModelCallContext {
+  return {
+    conversationId: "conv-scn",
+    callSite: "mainAgent",
+    content: [finalText],
+    messages: [],
+    stopReason: null,
+    decision: "stop",
+    logger: noopLogger,
+    ...overrides,
+  };
+}
+
+beforeEach(() => {
+  resetSurfaceCompletionNudgeStoreForTests();
+  surfaceCounter = 0;
+});
+
+// ─── Nudges when a progress surface is left open ──────────────────────────────
+
+describe("surface-completion-nudge — nudges on a dangling progress surface", () => {
+  test("task_progress card shown and never completed → continue with nudge", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("do the thing"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("continue");
+    const last = ctx.messages[ctx.messages.length - 1];
+    expect(last.role).toBe("user");
+    expect(last.content[0]).toEqual({
+      type: "text",
+      text: SURFACE_COMPLETION_NUDGE_TEXT,
+    });
+    expect(isSurfaceCompletionNudged("conv-scn")).toBe(true);
+  });
+
+  test("task_progress card shown with no explicit status → continue with nudge", async () => {
+    const shown = showSurface(taskProgressShow());
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("continue");
+  });
+
+  test("work_result shown in_progress → continue with nudge", async () => {
+    const shown = showSurface({
+      surface_type: "work_result",
+      data: { status: "in_progress", summary: "Crunching" },
+    });
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("continue");
+  });
+});
+
+// ─── Stays quiet when there is nothing to close ───────────────────────────────
+
+describe("surface-completion-nudge — no nudge when surface is closed or absent", () => {
+  test("task_progress completed via ui_update → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [
+        userPrompt("go"),
+        ...shown.messages,
+        updateSurface(shown.surfaceId, {
+          templateData: { status: "completed" },
+        }),
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+    expect(isSurfaceCompletionNudged("conv-scn")).toBe(false);
+  });
+
+  test("progress surface dismissed via ui_dismiss → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [
+        userPrompt("go"),
+        ...shown.messages,
+        dismissSurface(shown.surfaceId),
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("work_result shown completed → stop", async () => {
+    const shown = showSurface({
+      surface_type: "work_result",
+      data: { status: "completed", summary: "Done" },
+    });
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("plain card (not a progress surface) → stop", async () => {
+    const shown = showSurface({
+      surface_type: "card",
+      title: "Weather",
+      data: { template: "weather_forecast", body: "Sunny" },
+    });
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("no surfaces shown at all → stop", async () => {
+    const ctx = makeCtx({
+      messages: [userPrompt("go")],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+});
+
+// ─── Outcomes the hook does not own ───────────────────────────────────────────
+
+describe("surface-completion-nudge — ignores outcomes it does not own", () => {
+  test("provider rejection (error present) → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+      error: new Error("provider exploded"),
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("tool-bearing turn (model still working) → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+      content: [
+        { type: "tool_use", id: "tu_next", name: "read_file", input: {} },
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("non-main-agent call site → stop", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+      callSite: "heartbeatAgent",
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+});
+
+// ─── Cycle scoping and the one-shot bound ─────────────────────────────────────
+
+describe("surface-completion-nudge — cycle scoping and one-shot bound", () => {
+  test("a surface left open in a prior cycle does not trigger this cycle", async () => {
+    const priorOpen = showSurface(taskProgressShow("in_progress"));
+    const ctx = makeCtx({
+      messages: [
+        userPrompt("first task"),
+        ...priorOpen.messages,
+        // New genuine user prompt opens a fresh cycle with no open surface.
+        userPrompt("second task"),
+      ],
+    });
+
+    await postModelCall(ctx);
+
+    expect(ctx.decision).toBe("stop");
+  });
+
+  test("nudges at most once per run; stop clears the bound", async () => {
+    const shown = showSurface(taskProgressShow("in_progress"));
+    const firstCtx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+
+    await postModelCall(firstCtx);
+    expect(firstCtx.decision).toBe("continue");
+
+    // Same run, surface still open: the one-shot bound suppresses a second nudge.
+    const secondCtx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+    await postModelCall(secondCtx);
+    expect(secondCtx.decision).toBe("stop");
+
+    // Terminal stop clears the bound so the next run nudges afresh.
+    await stop({
+      conversationId: "conv-scn",
+      messages: [],
+      exitReason: "no_tool_calls",
+      logger: noopLogger,
+    });
+    expect(isSurfaceCompletionNudged("conv-scn")).toBe(false);
+
+    const thirdCtx = makeCtx({
+      messages: [userPrompt("go"), ...shown.messages],
+    });
+    await postModelCall(thirdCtx);
+    expect(thirdCtx.decision).toBe("continue");
+  });
+});

package/src/__tests__/system-storage-cleanup-skill.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, test } from "bun:test";
+
+import { loadSkillBySelector, loadSkillCatalog } from "../config/skills.js";
+
+describe("system-storage-cleanup bundled skill", () => {
+  test("is bundled without tools or inline command expansions", () => {
+    const catalog = loadSkillCatalog();
+    const skill = catalog.find(
+      (candidate) => candidate.id === "system-storage-cleanup",
+    );
+
+    expect(skill).toBeDefined();
+    expect(skill!.source).toBe("bundled");
+    expect(skill!.bundled).toBe(true);
+    expect(skill!.toolManifest).toBeUndefined();
+    expect(skill!.inlineCommandExpansions).toBeUndefined();
+  });
+
+  test("loads by id with the cleanup safety contract", () => {
+    const result = loadSkillBySelector("system-storage-cleanup");
+
+    expect(result.error).toBeUndefined();
+    expect(result.skill).toBeDefined();
+    expect(result.skill!.id).toBe("system-storage-cleanup");
+    expect(result.skill!.source).toBe("bundled");
+
+    const body = result.skill!.body.toLowerCase();
+
+    expect(body).toContain("normal work is suspended");
+    expect(body).toContain("stay scoped to freeing storage");
+    expect(body).toContain("prefer foreground inspection");
+    expect(body).toContain("before any mutation");
+    expect(body).toContain("ask for explicit approval before deleting");
+    expect(body).toContain("target volume");
+    expect(body).toContain("workspace path");
+    expect(body).toContain("read-only `sqlite3` access");
+    expect(body).toContain("never run ad hoc `delete`");
+    expect(body).toContain("`vacuum`");
+    expect(body).toContain("mutating sqlite command");
+
+    for (const protectedText of [
+      "credentials",
+      "security material",
+      "workspace database files",
+      "config files",
+      "backups",
+      "backup keys",
+      "memory graph nodes or segments",
+      "`journal/`",
+      "`data/reflections/`",
+      "pkb files",
+    ]) {
+      expect(body).toContain(protectedText);
+    }
+  });
+});

package/src/__tests__/task-progress-nudge-hook.test.ts

@@ -104,7 +104,7 @@ function currentResponse(toolUseId: string): ToolResultContent {
   };
 }
 
-/** A weak-model id that matches WEAK_MODEL_PATTERN (the gated population). */
+/** A weak-model id that matches WEAK_OPEN_MODEL_PATTERN (the gated population). */
 const WEAK_MODEL = "minimax/minimax-m3";
 
 function makeCtx(

package/src/__tests__/title-generate-hook.test.ts

@@ -7,8 +7,8 @@
  * - `user-prompt-submit` — first-pass generation from the submitted prompt,
  *   scheduled on a later macrotask so the main agent-loop LLM request is
  *   issued first.
- * - `stop` — second-pass regeneration once the conversation reaches its third
- *   user turn (turn count derived from the user prompts in history).
+ * - `stop` — fallback retry for replaceable titles and second-pass
+ *   regeneration once the conversation reaches its third user turn.
  *
  * Both let the title service resolve the provider, persist the title, and
  * broadcast the resulting `conversation_title_updated` / `sync_changed`
@@ -31,13 +31,42 @@ const queueGenerateConversationTitleMock = mock(
   }): void => undefined,
 );
 const queueRegenerateConversationTitleMock = mock(
-  (_params: { conversationId: string; provider?: unknown }): void => undefined,
+  (_params: {
+    conversationId: string;
+    provider?: unknown;
+    onlyIfReplaceable?: boolean;
+  }): void => undefined,
 );
 mock.module("../memory/conversation-title-service.js", () => ({
+  AUTO_TITLE_DETERMINISTIC: 2,
+  isReplaceableTitle: (title: string | null) =>
+    title == null ||
+    title === "" ||
+    title === "Generating title..." ||
+    title === "New Conversation" ||
+    title === "Untitled" ||
+    title === "Untitled Conversation" ||
+    title.startsWith("Runtime: "),
   queueGenerateConversationTitle: queueGenerateConversationTitleMock,
   queueRegenerateConversationTitle: queueRegenerateConversationTitleMock,
 }));
 
+const mockGetConversation = mock(
+  (_conversationId: string) =>
+    ({
+      title: "Existing Title",
+      isAutoTitle: 1,
+      conversationType: "standard",
+    }) as {
+      title: string;
+      isAutoTitle: number;
+      conversationType: string;
+    },
+);
+mock.module("../memory/conversation-crud.js", () => ({
+  getConversation: mockGetConversation,
+}));
+
 // The `stop` hook reads `conversations.skipAutoRetitling`; stub the loader so
 // the opt-out is controllable per test.
 let skipAutoRetitling = false;
@@ -192,6 +221,19 @@ describe("title-generate stop hook", () => {
     resetPluginRegistryForTests();
     queueRegenerateConversationTitleMock.mockReset();
     queueRegenerateConversationTitleMock.mockImplementation(() => undefined);
+    mockGetConversation.mockReset();
+    mockGetConversation.mockImplementation(
+      (_conversationId: string) =>
+        ({
+          title: "Existing Title",
+          isAutoTitle: 1,
+          conversationType: "standard",
+        }) as {
+          title: string;
+          isAutoTitle: number;
+          conversationType: string;
+        },
+    );
     skipAutoRetitling = false;
   });
 
@@ -210,6 +252,61 @@ describe("title-generate stop hook", () => {
     expect(call?.conversationId).toBe("conv-1");
     expect(call).not.toHaveProperty("provider");
     expect(call).not.toHaveProperty("signal");
+    expect(call).not.toHaveProperty("onlyIfReplaceable");
+  });
+
+  test("retries a replaceable fallback title after a successful turn", async () => {
+    mockGetConversation.mockReturnValueOnce({
+      title: "Untitled Conversation",
+      isAutoTitle: 2,
+      conversationType: "standard",
+    });
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(1) });
+
+    await stop(ctx);
+    await flushMacrotasks();
+
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledWith({
+      conversationId: "conv-1",
+      onlyIfReplaceable: true,
+    });
+  });
+
+  test("preserves the third-turn retitle when the title is still replaceable", async () => {
+    mockGetConversation.mockReturnValueOnce({
+      title: "Untitled Conversation",
+      isAutoTitle: 2,
+      conversationType: "standard",
+    });
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    await stop(ctx);
+    await flushMacrotasks();
+
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    const call = queueRegenerateConversationTitleMock.mock.calls[0]?.[0];
+    expect(call?.conversationId).toBe("conv-1");
+    expect(call).not.toHaveProperty("onlyIfReplaceable");
+  });
+
+  test("fallback title retry is not blocked by the retitling opt-out", async () => {
+    skipAutoRetitling = true;
+    mockGetConversation.mockReturnValueOnce({
+      title: "Generating title...",
+      isAutoTitle: 1,
+      conversationType: "standard",
+    });
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(1) });
+
+    await stop(ctx);
+    await flushMacrotasks();
+
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledWith({
+      conversationId: "conv-1",
+      onlyIfReplaceable: true,
+    });
   });
 
   test("defers the regeneration so the completed turn is persisted first", async () => {

package/src/__tests__/token-estimator-accuracy.benchmark.test.ts

@@ -128,7 +128,6 @@ function makeSystemPrompt(size: "small" | "production" = "small"): string {
     "- `file_write` — Creating or overwriting files",
     "- `file_edit` — Modifying existing files",
     "- `file_delete` — Deleting files",
-    "- `credential_store set` — Storing credentials",
     "",
     "### Medium-Risk Tools (require approval on first use per session)",
     "- `web_fetch` — Fetching external URLs",
@@ -349,7 +348,7 @@ function makeToolDefinitions(): Array<{
     input_schema: object;
   }> = [];
 
-  // Core tools (11)
+  // Core tools (10)
   tools.push(
     {
       name: "bash",
@@ -566,33 +565,6 @@ function makeToolDefinitions(): Array<{
         required: ["query"],
       },
     },
-    {
-      name: "credential_store",
-      description:
-        "Securely store or retrieve credentials for external services. Credentials are encrypted at rest.",
-      input_schema: {
-        type: "object",
-        properties: {
-          action: {
-            type: "string",
-            enum: ["get", "set", "delete", "list"],
-          },
-          service: {
-            type: "string",
-            description: "The service name (e.g., 'github', 'slack')",
-          },
-          key: {
-            type: "string",
-            description: "Credential key within the service",
-          },
-          value: {
-            type: "string",
-            description: "Credential value (required for 'set')",
-          },
-        },
-        required: ["action", "service"],
-      },
-    },
   );
 
   // Computer-use proxy tools (11)