@vellumai/assistant 0.8.7 → 0.8.8-dev.202606052332.17fc8ea

package/src/__tests__/task-scheduler.test.ts

@@ -24,6 +24,7 @@ mock.module("../runtime/background-job-runner.js", () => ({
     prompt: string;
     groupId?: string;
     trustContext: { sourceChannel: string; trustClass: string };
+    onConversationCreated?: (conversationId: string) => void;
   }) => {
     const { createConversation } =
       await import("../memory/conversation-crud.js");
@@ -33,6 +34,7 @@ mock.module("../runtime/background-job-runner.js", () => ({
       source: "schedule",
       ...(opts.groupId ? { groupId: opts.groupId } : {}),
     });
+    opts.onConversationCreated?.(conv.id);
     onRunBackgroundJobCall?.({
       conversationId: conv.id,
       prompt: opts.prompt,
@@ -60,12 +62,17 @@ mock.module("../notifications/emit-signal.js", () => ({
 
 import { getDb } from "../memory/db-connection.js";
 import { initializeDb } from "../memory/db-init.js";
+import { recordUsageEvent } from "../memory/llm-usage-store.js";
 import {
   createSchedule,
   getSchedule,
   getScheduleRuns,
 } from "../schedule/schedule-store.js";
-import { startScheduler } from "../schedule/scheduler.js";
+import { getScheduleUsageSummaries } from "../schedule/schedule-usage-store.js";
+import {
+  runScheduleDueWorkOnce,
+  startScheduler,
+} from "../schedule/scheduler.js";
 import { scheduleTask } from "../tasks/task-scheduler.js";
 import { createTask } from "../tasks/task-store.js";
 
@@ -160,6 +167,7 @@ describe("scheduler run_task detection", () => {
     db.run("DELETE FROM cron_jobs");
     db.run("DELETE FROM task_runs");
     db.run("DELETE FROM tasks");
+    db.run("DELETE FROM llm_usage_events");
     db.run("DELETE FROM messages");
     db.run("DELETE FROM conversations");
     onRunBackgroundJobCall = null;
@@ -298,4 +306,157 @@ describe("scheduler run_task detection", () => {
       /^activity-failed:task:nonexistent-task-id:\d{4}-\d{2}-\d{2}$/,
     );
   });
+
+  test("opens a task-backed schedule run before task processing and backfills the real conversation id", async () => {
+    const task = createTask({
+      title: "Usage Attribution Task",
+      template: "Spend scheduled tokens",
+    });
+    const schedule = scheduleTask({
+      taskId: task.id,
+      name: "Usage Attribution Schedule",
+      cronExpression: "* * * * *",
+    });
+    forceScheduleDue(schedule.id);
+
+    const from = Date.now() - 1000;
+    let processingConversationId: string | null = null;
+    let usageEventCreatedAt: number | null = null;
+    let runsDuringProcessing: ReturnType<typeof getScheduleRuns> = [];
+
+    const result = await runScheduleDueWorkOnce(
+      async (conversationId) => {
+        processingConversationId = conversationId;
+        runsDuringProcessing = getScheduleRuns(schedule.id);
+        const event = recordUsageEvent(
+          {
+            conversationId,
+            runId: null,
+            requestId: "req-scheduled-task-usage",
+            actor: "main_agent",
+            callSite: "mainAgent",
+            inferenceProfile: "balanced",
+            provider: "anthropic",
+            model: "claude-sonnet-4-20250514",
+            inputTokens: 100,
+            outputTokens: 50,
+            cacheCreationInputTokens: 0,
+            cacheReadInputTokens: 0,
+            rawUsage: null,
+          },
+          { estimatedCostUsd: 0.25, pricingStatus: "priced" },
+        );
+        usageEventCreatedAt = event.createdAt;
+      },
+      () => {},
+    );
+    const to = Date.now() + 1000;
+
+    expect(result.completed).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(processingConversationId).not.toBeNull();
+    expect(usageEventCreatedAt).not.toBeNull();
+    expect(runsDuringProcessing).toHaveLength(1);
+    expect(runsDuringProcessing[0].status).toBe("running");
+    expect(runsDuringProcessing[0].conversationId).toBeNull();
+    expect(runsDuringProcessing[0].startedAt).toBeLessThanOrEqual(
+      usageEventCreatedAt!,
+    );
+
+    const runs = getScheduleRuns(schedule.id);
+    expect(runs).toHaveLength(1);
+    expect(runs[0].id).toBe(runsDuringProcessing[0].id);
+    expect(runs[0].status).toBe("ok");
+    expect(runs[0].conversationId).toBe(processingConversationId);
+    expect(runs[0].startedAt).toBeLessThanOrEqual(usageEventCreatedAt!);
+    expect(runs[0].finishedAt).not.toBeNull();
+    expect(runs[0].finishedAt!).toBeGreaterThanOrEqual(usageEventCreatedAt!);
+
+    const summary = getScheduleUsageSummaries({ from, to }).find(
+      (row) => row.scheduleId === schedule.id,
+    );
+    expect(summary).toEqual({
+      scheduleId: schedule.id,
+      runCount: 1,
+      totalEstimatedCostUsd: 0.25,
+      eventCount: 1,
+    });
+  });
+
+  test("opens a normal execute schedule run before fresh background processing and records the conversation id", async () => {
+    const schedule = createSchedule({
+      name: "Usage Attribution Message Schedule",
+      cronExpression: "* * * * *",
+      message: "Spend scheduled message tokens",
+      syntax: "cron",
+    });
+    forceScheduleDue(schedule.id);
+
+    const from = Date.now() - 1000;
+    let processingConversationId: string | null = null;
+    let usageEventCreatedAt: number | null = null;
+    let runsDuringProcessing: ReturnType<typeof getScheduleRuns> = [];
+
+    onRunBackgroundJobCall = (info) => {
+      processingConversationId = info.conversationId;
+      runsDuringProcessing = getScheduleRuns(schedule.id);
+      const event = recordUsageEvent(
+        {
+          conversationId: info.conversationId,
+          runId: null,
+          requestId: "req-scheduled-message-usage",
+          actor: "main_agent",
+          callSite: "mainAgent",
+          inferenceProfile: "balanced",
+          provider: "anthropic",
+          model: "claude-sonnet-4-20250514",
+          inputTokens: 80,
+          outputTokens: 20,
+          cacheCreationInputTokens: 0,
+          cacheReadInputTokens: 0,
+          rawUsage: null,
+        },
+        { estimatedCostUsd: 0.1, pricingStatus: "priced" },
+      );
+      usageEventCreatedAt = event.createdAt;
+    };
+
+    const result = await runScheduleDueWorkOnce(
+      async () => {},
+      () => {},
+    );
+    const to = Date.now() + 1000;
+
+    expect(result.completed).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(processingConversationId).not.toBeNull();
+    expect(usageEventCreatedAt).not.toBeNull();
+    expect(runsDuringProcessing).toHaveLength(1);
+    expect(runsDuringProcessing[0].status).toBe("running");
+    expect(runsDuringProcessing[0].conversationId).toBe(
+      processingConversationId,
+    );
+    expect(runsDuringProcessing[0].startedAt).toBeLessThanOrEqual(
+      usageEventCreatedAt!,
+    );
+
+    const runs = getScheduleRuns(schedule.id);
+    expect(runs).toHaveLength(1);
+    expect(runs[0].id).toBe(runsDuringProcessing[0].id);
+    expect(runs[0].status).toBe("ok");
+    expect(runs[0].conversationId).toBe(processingConversationId);
+    expect(runs[0].startedAt).toBeLessThanOrEqual(usageEventCreatedAt!);
+    expect(runs[0].finishedAt).not.toBeNull();
+    expect(runs[0].finishedAt!).toBeGreaterThanOrEqual(usageEventCreatedAt!);
+
+    const summary = getScheduleUsageSummaries({ from, to }).find(
+      (row) => row.scheduleId === schedule.id,
+    );
+    expect(summary).toEqual({
+      scheduleId: schedule.id,
+      runCount: 1,
+      totalEstimatedCostUsd: 0.1,
+      eventCount: 1,
+    });
+  });
 });

package/src/__tests__/terminal-tools.test.ts

@@ -141,7 +141,7 @@ describe("buildSanitizedEnv", () => {
     expect(env.LC_CTYPE).toBe("UTF-8");
   });
 
-  test("only includes Kata apt variables when sandbox runtime is kata", () => {
+  test("only includes Kata apt variables for Kata-family sandbox runtimes", () => {
     process.env.VELLUM_SANDBOX_RUNTIME = "gvisor";
     process.env.PATH = "/usr/bin";
     process.env.VELLUM_APT_DATA_ROOT = "/data/system";
@@ -162,6 +162,11 @@ describe("buildSanitizedEnv", () => {
       "/data/system/usr/local/lib",
     );
     expect(env.LD_LIBRARY_PATH.split(":")).not.toContain("/host/lib");
+
+    process.env.VELLUM_SANDBOX_RUNTIME = "cloud-hypervisor";
+    env = buildSanitizedEnv();
+    expect(env.VELLUM_APT_DATA_ROOT).toBe("/data/system");
+    expect(env.PATH.split(":")).toContain("/data/system/usr/bin");
   });
 
   test("defaults LANG and LC_ALL to UTF-8 when unset", () => {

package/src/__tests__/title-generate-hook.test.ts

@@ -0,0 +1,319 @@
+/**
+ * Tests for the default `title-generate` plugin's hooks.
+ *
+ * The plugin contributes two pure-trigger hooks that delegate the title work
+ * to the service:
+ *
+ * - `user-prompt-submit` — first-pass generation from the submitted prompt,
+ *   scheduled on a later macrotask so the main agent-loop LLM request is
+ *   issued first.
+ * - `stop` — second-pass regeneration once the conversation reaches its third
+ *   user turn (turn count derived from the user prompts in history).
+ *
+ * Both let the title service resolve the provider, persist the title, and
+ * broadcast the resulting `conversation_title_updated` / `sync_changed`
+ * events.
+ *
+ * Mocks `memory/conversation-title-service.js` and `config/loader.js` so the
+ * tests don't touch the real provider stack or config, and resets the plugin
+ * registry between cases.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// Stub the title-generation service before importing anything that binds
+// to it, so both the default plugin and the hooks capture the stubbed binding.
+const queueGenerateConversationTitleMock = mock(
+  (_params: {
+    conversationId: string;
+    provider?: unknown;
+    userMessage?: string;
+  }): void => undefined,
+);
+const queueRegenerateConversationTitleMock = mock(
+  (_params: { conversationId: string; provider?: unknown }): void => undefined,
+);
+mock.module("../memory/conversation-title-service.js", () => ({
+  queueGenerateConversationTitle: queueGenerateConversationTitleMock,
+  queueRegenerateConversationTitle: queueRegenerateConversationTitleMock,
+}));
+
+// The `stop` hook reads `conversations.skipAutoRetitling`; stub the loader so
+// the opt-out is controllable per test.
+let skipAutoRetitling = false;
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ conversations: { skipAutoRetitling } }),
+}));
+
+import { HOOKS } from "../plugin-api/constants.js";
+import type {
+  PluginLogger,
+  StopContext,
+  UserPromptSubmitContext,
+} from "../plugin-api/types.js";
+import stop from "../plugins/defaults/title-generate/hooks/stop.js";
+import userPromptSubmit from "../plugins/defaults/title-generate/hooks/user-prompt-submit.js";
+import { defaultTitleGeneratePlugin } from "../plugins/defaults/title-generate/register.js";
+import { runHook } from "../plugins/pipeline.js";
+import {
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
+import type { Message } from "../providers/types.js";
+
+const noopLogger: PluginLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+
+function makeCtx(
+  overrides: Partial<UserPromptSubmitContext> = {},
+): UserPromptSubmitContext {
+  const messages: Message[] = [
+    { role: "user", content: [{ type: "text", text: "first message" }] },
+  ];
+  return {
+    conversationId: "conv-1",
+    prompt: "first message",
+    originalMessages: messages,
+    latestMessages: messages,
+    logger: noopLogger,
+    ...overrides,
+  };
+}
+
+/** Flush pending `setTimeout(0)` callbacks so the fire-and-forget trigger runs. */
+function flushMacrotasks(): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, 0));
+}
+
+function userTurn(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+function assistantTurn(text: string): Message {
+  return { role: "assistant", content: [{ type: "text", text }] };
+}
+
+/** A user-role message carrying only tool results, not a fresh prompt. */
+function toolResultTurn(): Message {
+  return {
+    role: "user",
+    content: [{ type: "tool_result", tool_use_id: "tool-1", content: "ok" }],
+  };
+}
+
+/** History with `count` genuine user turns interleaved with assistant replies. */
+function historyWithUserTurns(count: number): Message[] {
+  const messages: Message[] = [];
+  for (let i = 1; i <= count; i++) {
+    messages.push(userTurn(`message ${i}`));
+    messages.push(assistantTurn(`reply ${i}`));
+  }
+  return messages;
+}
+
+function makeStopCtx(overrides: Partial<StopContext> = {}): StopContext {
+  return {
+    conversationId: "conv-1",
+    messages: historyWithUserTurns(3),
+    responseContent: [{ type: "text", text: "reply 3" }],
+    stopReason: "end_turn",
+    decision: "stop",
+    logger: noopLogger,
+    ...overrides,
+  };
+}
+
+describe("title-generate user-prompt-submit hook", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+    queueGenerateConversationTitleMock.mockReset();
+    queueGenerateConversationTitleMock.mockImplementation(() => undefined);
+  });
+
+  test("queues a title-generation job from the submitted prompt", async () => {
+    // GIVEN a fresh user prompt submission
+    const ctx = makeCtx({ conversationId: "conv-1", prompt: "first message" });
+
+    // WHEN the default hook runs and its deferred work flushes
+    await userPromptSubmit(ctx);
+    await flushMacrotasks();
+
+    // THEN the title service is invoked with just the conversation id and the
+    // submitted prompt — provider resolution and emit are owned by the service.
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    const call = queueGenerateConversationTitleMock.mock.calls[0]?.[0];
+    expect(call?.conversationId).toBe("conv-1");
+    expect(call?.userMessage).toBe("first message");
+    expect(call).not.toHaveProperty("provider");
+    expect(call).not.toHaveProperty("onTitleUpdated");
+  });
+
+  test("does not block: returns before the title job is scheduled", async () => {
+    // GIVEN a fresh user prompt submission
+    const ctx = makeCtx();
+
+    // WHEN the hook resolves
+    await userPromptSubmit(ctx);
+
+    // THEN the title job has not run yet (it is deferred to a later macrotask),
+    // AND it runs once the macrotask queue is flushed.
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+    await flushMacrotasks();
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("fires through runHook once the default plugin is registered", async () => {
+    // GIVEN the default title-generate plugin registered in the registry
+    registerPlugin(defaultTitleGeneratePlugin);
+
+    // WHEN a prompt is submitted through the hook chain
+    await runHook(
+      HOOKS.USER_PROMPT_SUBMIT,
+      makeCtx({ prompt: "draft a plan" }),
+    );
+    await flushMacrotasks();
+
+    // THEN the title service is triggered with the submitted prompt text
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    expect(
+      queueGenerateConversationTitleMock.mock.calls[0]?.[0]?.userMessage,
+    ).toBe("draft a plan");
+  });
+});
+
+describe("title-generate stop hook", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+    queueRegenerateConversationTitleMock.mockReset();
+    queueRegenerateConversationTitleMock.mockImplementation(() => undefined);
+    skipAutoRetitling = false;
+  });
+
+  test("regenerates the title on the third user turn", async () => {
+    // GIVEN a turn ending with three genuine user prompts in history
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    // WHEN the stop hook runs and its deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN the second-pass regeneration is triggered with just the
+    // conversation id — provider resolution and emit are owned by the service.
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    const call = queueRegenerateConversationTitleMock.mock.calls[0]?.[0];
+    expect(call?.conversationId).toBe("conv-1");
+    expect(call).not.toHaveProperty("provider");
+    expect(call).not.toHaveProperty("signal");
+  });
+
+  test("defers the regeneration so the completed turn is persisted first", async () => {
+    // GIVEN a turn ending on the third user turn
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    // WHEN the hook resolves
+    await stop(ctx);
+
+    // THEN the regeneration has not fired yet — it is deferred to a later
+    // macrotask so the turn's assistant reply lands first, AND it fires once
+    // the macrotask queue is flushed.
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+    await flushMacrotasks();
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("does not regenerate before the third user turn", async () => {
+    // GIVEN a turn ending with only two genuine user prompts
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(2) });
+
+    // WHEN the stop hook runs and any deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN no regeneration fires — the conversation lacks enough context yet
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("does not regenerate after the third user turn", async () => {
+    // GIVEN a turn ending with four genuine user prompts
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(4) });
+
+    // WHEN the stop hook runs and any deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN no regeneration fires — the single second pass already passed
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("ignores tool-result user messages when counting turns", async () => {
+    // GIVEN three genuine user prompts plus a tool-result user message
+    const messages: Message[] = [
+      userTurn("message 1"),
+      assistantTurn("reply 1"),
+      userTurn("message 2"),
+      assistantTurn("calling a tool"),
+      toolResultTurn(),
+      assistantTurn("reply 2"),
+      userTurn("message 3"),
+    ];
+    const ctx = makeStopCtx({ messages });
+
+    // WHEN the stop hook runs and its deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN the tool-result message is not counted as a turn, so the third
+    // genuine prompt still triggers the regeneration
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("defers when an earlier hook chose to continue the run", async () => {
+    // GIVEN a third-user-turn stop that an earlier hook flipped to "continue"
+    const ctx = makeStopCtx({
+      messages: historyWithUserTurns(3),
+      decision: "continue",
+    });
+
+    // WHEN the stop hook runs and any deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN it defers to the eventual terminal stop rather than re-titling now
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("respects the skipAutoRetitling opt-out", async () => {
+    // GIVEN the user opted out of second-pass retitling
+    skipAutoRetitling = true;
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    // WHEN the stop hook runs on the third user turn and any work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN no regeneration fires
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("fires through runHook once the default plugin is registered", async () => {
+    // GIVEN the default title-generate plugin registered in the registry
+    registerPlugin(defaultTitleGeneratePlugin);
+
+    // WHEN a third-user-turn stop is dispatched through the hook chain
+    await runHook(
+      HOOKS.STOP,
+      makeStopCtx({ messages: historyWithUserTurns(3) }),
+    );
+    await flushMacrotasks();
+
+    // THEN the second-pass regeneration is triggered
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    expect(
+      queueRegenerateConversationTitleMock.mock.calls[0]?.[0]?.conversationId,
+    ).toBe("conv-1");
+  });
+});