@vellumai/assistant 0.8.7 → 0.8.8

package/src/__tests__/usage-routes.test.ts

@@ -9,13 +9,18 @@ mock.module("../util/logger.js", () => ({
 
 import { getSqlite } from "../memory/db-connection.js";
 import { initializeDb } from "../memory/db-init.js";
-import { recordUsageEvent } from "../memory/llm-usage-store.js";
+import {
+  getUsageCostForConversationWindow,
+  recordUsageEvent,
+} from "../memory/llm-usage-store.js";
 import { BadRequestError } from "../runtime/routes/errors.js";
 import { ROUTES } from "../runtime/routes/usage-routes.js";
 
 initializeDb();
 
 function clearUsageEvents() {
+  getSqlite().run("DELETE FROM cron_runs");
+  getSqlite().run("DELETE FROM cron_jobs");
   getSqlite().run("DELETE FROM llm_usage_events");
 }
 
@@ -118,6 +123,107 @@ function seedEvents() {
   return { day1, day2 };
 }
 
+function recordCostAt(
+  conversationId: string,
+  requestId: string,
+  createdAt: number,
+  estimatedCostUsd: number,
+) {
+  recordUsageEvent(
+    {
+      conversationId,
+      runId: null,
+      requestId,
+      actor: "main_agent",
+      callSite: "mainAgent",
+      inferenceProfile: "balanced",
+      provider: "anthropic",
+      model: "claude-sonnet-4-20250514",
+      inputTokens: 100,
+      outputTokens: 50,
+      cacheCreationInputTokens: 0,
+      cacheReadInputTokens: 0,
+      rawUsage: null,
+    },
+    { estimatedCostUsd, pricingStatus: "priced" },
+  );
+  getSqlite().run(
+    "UPDATE llm_usage_events SET created_at = ? WHERE request_id = ?",
+    [createdAt, requestId],
+  );
+}
+
+function insertScheduleJob(id: string, name: string): void {
+  const now = new Date("2026-01-01T00:00:00Z").getTime();
+  getSqlite().run(
+    `INSERT INTO cron_jobs (
+      id,
+      name,
+      cron_expression,
+      message,
+      next_run_at,
+      created_by,
+      created_at,
+      updated_at
+    ) VALUES (?, ?, '* * * * *', 'Example scheduled task', ?, 'user', ?, ?)`,
+    [id, name, now, now, now],
+  );
+}
+
+function insertScheduleRun({
+  id,
+  scheduleId,
+  conversationId,
+  startedAt,
+  finishedAt,
+}: {
+  id: string;
+  scheduleId: string;
+  conversationId: string;
+  startedAt: number;
+  finishedAt: number | null;
+}): void {
+  getSqlite().run(
+    `INSERT INTO cron_runs (
+      id,
+      job_id,
+      status,
+      started_at,
+      finished_at,
+      conversation_id,
+      created_at
+    ) VALUES (?, ?, 'ok', ?, ?, ?, ?)`,
+    [id, scheduleId, startedAt, finishedAt, conversationId, startedAt],
+  );
+}
+
+function seedScheduleRouteEvents() {
+  insertScheduleJob("schedule-a", "Morning summary");
+  insertScheduleJob("schedule-b", "Nightly sync");
+  insertScheduleRun({
+    id: "run-a-1",
+    scheduleId: "schedule-a",
+    conversationId: "conv-reused",
+    startedAt: 1_000,
+    finishedAt: 2_000,
+  });
+  insertScheduleRun({
+    id: "run-b-1",
+    scheduleId: "schedule-b",
+    conversationId: "conv-reused",
+    startedAt: 3_000,
+    finishedAt: 3_500,
+  });
+
+  recordCostAt("conv-reused", "route-before-a", 900, 0.09);
+  recordCostAt("conv-reused", "route-a-start", 1_000, 0.1);
+  recordCostAt("conv-reused", "route-a-inside", 1_500, 0.2);
+  recordCostAt("conv-reused", "route-a-finish", 2_000, 0.3);
+  recordCostAt("conv-reused", "route-after-a", 2_500, 0.4);
+  recordCostAt("conv-reused", "route-b-inside", 3_200, 0.5);
+  recordCostAt("conv-other", "route-other", 1_500, 0.8);
+}
+
 // ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
@@ -125,6 +231,25 @@ function seedEvents() {
 describe("usage routes", () => {
   beforeEach(clearUsageEvents);
 
+  describe("getUsageCostForConversationWindow", () => {
+    test("sums only events for the conversation inside the inclusive window", () => {
+      recordCostAt("conv-window", "req-before", 999, 0.5);
+      recordCostAt("conv-window", "req-start", 1000, 0.01);
+      recordCostAt("conv-window", "req-middle", 1500, 0.02);
+      recordCostAt("conv-window", "req-end", 2000, 0.03);
+      recordCostAt("conv-window", "req-after", 2001, 0.75);
+      recordCostAt("conv-other", "req-other", 1500, 0.9);
+
+      const total = getUsageCostForConversationWindow({
+        conversationId: "conv-window",
+        from: 1000,
+        to: 2000,
+      });
+
+      expect(total).toBeCloseTo(0.06);
+    });
+  });
+
   // -- query parsing / validation --
 
   describe("query parameter validation", () => {
@@ -204,6 +329,19 @@ describe("usage routes", () => {
       expect(body.totalCacheCreationTokens).toBe(50);
       expect(body.totalCacheReadTokens).toBe(100);
     });
+
+    test("filters by trimmed scheduleId using schedule run windows", () => {
+      seedScheduleRouteEvents();
+
+      const body = dispatch(
+        "GET",
+        "usage/totals?from=0&to=4000&scheduleId=%20schedule-a%20",
+      ) as Record<string, number>;
+
+      expect(body.eventCount).toBe(3);
+      expect(body.totalInputTokens).toBe(300);
+      expect(body.totalEstimatedCostUsd).toBeCloseTo(0.6);
+    });
   });
 
   // -- daily buckets --
@@ -255,6 +393,21 @@ describe("usage routes", () => {
       expect(body.buckets[1].totalInputTokens).toBe(2000);
       expect(body.buckets[1].eventCount).toBe(1);
     });
+
+    test("filters daily buckets by scheduleId", () => {
+      seedScheduleRouteEvents();
+
+      const body = dispatch(
+        "GET",
+        "usage/daily?from=0&to=4000&scheduleId=schedule-a",
+      ) as {
+        buckets: Array<{ totalEstimatedCostUsd: number; eventCount: number }>;
+      };
+
+      expect(body.buckets).toHaveLength(1);
+      expect(body.buckets[0].eventCount).toBe(3);
+      expect(body.buckets[0].totalEstimatedCostUsd).toBeCloseTo(0.6);
+    });
   });
 
   // -- breakdown --
@@ -396,6 +549,70 @@ describe("usage routes", () => {
         null,
       ]);
     });
+
+    test("accepts groupBy=schedule and labels groups with schedule names", () => {
+      seedScheduleRouteEvents();
+
+      const body = dispatch(
+        "GET",
+        "usage/breakdown?from=0&to=4000&groupBy=schedule",
+      ) as {
+        breakdown: Array<{
+          group: string;
+          groupId: string | null;
+          groupKey: string | null;
+          totalEstimatedCostUsd: number;
+          eventCount: number;
+        }>;
+      };
+
+      expect(
+        body.breakdown.find((row) => row.groupKey === "schedule-a"),
+      ).toMatchObject({
+        group: "Morning summary",
+        groupId: "schedule-a",
+        totalEstimatedCostUsd: 0.6,
+        eventCount: 3,
+      });
+      expect(
+        body.breakdown.find((row) => row.groupKey === "schedule-b"),
+      ).toMatchObject({
+        group: "Nightly sync",
+        groupId: "schedule-b",
+        totalEstimatedCostUsd: 0.5,
+        eventCount: 1,
+      });
+      expect(body.breakdown.find((row) => row.groupKey === null)).toMatchObject(
+        {
+          group: "Other",
+          groupId: null,
+          totalEstimatedCostUsd: 1.29,
+          eventCount: 3,
+        },
+      );
+    });
+
+    test("filters breakdown by scheduleId", () => {
+      seedScheduleRouteEvents();
+
+      const body = dispatch(
+        "GET",
+        "usage/breakdown?from=0&to=4000&groupBy=provider&scheduleId=schedule-a",
+      ) as {
+        breakdown: Array<{
+          group: string;
+          totalEstimatedCostUsd: number;
+          eventCount: number;
+        }>;
+      };
+
+      expect(body.breakdown).toHaveLength(1);
+      expect(body.breakdown[0]).toMatchObject({
+        group: "anthropic",
+        totalEstimatedCostUsd: 0.6,
+        eventCount: 3,
+      });
+    });
   });
 
   describe("GET /v1/usage/series", () => {
@@ -484,5 +701,72 @@ describe("usage routes", () => {
         totalInputTokens: 2000,
       });
     });
+
+    test("accepts groupBy=schedule and labels schedule series groups", () => {
+      seedScheduleRouteEvents();
+
+      const body = dispatch(
+        "GET",
+        "usage/series?from=0&to=4000&groupBy=schedule&granularity=daily",
+      ) as {
+        buckets: Array<{
+          groups: Record<
+            string,
+            {
+              group: string;
+              groupKey: string | null;
+              totalEstimatedCostUsd: number;
+            }
+          >;
+        }>;
+      };
+
+      expect(body.buckets).toHaveLength(1);
+      expect(body.buckets[0].groups["value:schedule-a"]).toMatchObject({
+        group: "Morning summary",
+        groupKey: "schedule-a",
+      });
+      expect(
+        body.buckets[0].groups["value:schedule-a"].totalEstimatedCostUsd,
+      ).toBeCloseTo(0.6);
+      expect(body.buckets[0].groups["value:schedule-b"]).toMatchObject({
+        group: "Nightly sync",
+        groupKey: "schedule-b",
+      });
+      expect(
+        body.buckets[0].groups["value:schedule-b"].totalEstimatedCostUsd,
+      ).toBeCloseTo(0.5);
+      expect(body.buckets[0].groups["null:schedule"]).toMatchObject({
+        group: "Other",
+        groupKey: null,
+      });
+      expect(
+        body.buckets[0].groups["null:schedule"].totalEstimatedCostUsd,
+      ).toBeCloseTo(1.29);
+    });
+
+    test("filters grouped series by scheduleId", () => {
+      seedScheduleRouteEvents();
+
+      const body = dispatch(
+        "GET",
+        "usage/series?from=0&to=4000&groupBy=call_site&granularity=daily&scheduleId=schedule-a",
+      ) as {
+        buckets: Array<{
+          totalEstimatedCostUsd: number;
+          groups: Record<string, { totalEstimatedCostUsd: number }>;
+        }>;
+      };
+
+      expect(body.buckets).toHaveLength(1);
+      expect(body.buckets[0].totalEstimatedCostUsd).toBeCloseTo(0.6);
+      expect(body.buckets[0].groups["value:mainAgent"]).toMatchObject({
+        group: "Main Agent",
+        groupKey: "mainAgent",
+      });
+      expect(
+        body.buckets[0].groups["value:mainAgent"].totalEstimatedCostUsd,
+      ).toBeCloseTo(0.6);
+    });
   });
 });

package/src/__tests__/user-plugin-loader.test.ts

@@ -181,8 +181,8 @@ registerPlugin({
     // continues running in the background and would otherwise call
     // `registerPlugin()` after `loadUserPlugins()` has returned (and after
     // `bootstrapPlugins()` has potentially already walked the registry),
-    // leaving the plugin visible to `getMiddlewaresFor()` / `getInjectors()`
-    // with its `init()` hook never invoked.
+    // leaving the plugin visible to `getMiddlewaresFor()` with its `init()`
+    // hook never invoked.
     //
     // The `closeRegistration()` latch must reject that late arrival so the
     // registry stays consistent with the bootstrap invariant.

package/src/__tests__/voice-session-bridge.test.ts

@@ -96,10 +96,10 @@ function makePersistingStreamingSession(
 
   let turnChannelContext: TurnChannelContext | null = null;
   let turnInterfaceContext: TurnInterfaceContext | null = null;
+  let processing = false;
   const session = {
     conversationId,
     messages: [],
-    processing: false,
     abortController: null,
     currentRequestId: undefined,
     queue: {} as never,
@@ -108,7 +108,10 @@ function makePersistingStreamingSession(
       scopeId: "default",
       includeDefaultFallback: false,
     },
-    isProcessing: () => session.processing,
+    isProcessing: () => processing,
+    setProcessing: (value: boolean) => {
+      processing = value;
+    },
     persistUserMessage: async (
       ...args: Parameters<Conversation["persistUserMessage"]>
     ) => persistUserMessageImpl(session, ...args),
@@ -138,7 +141,7 @@ function makePersistingStreamingSession(
       for (const event of events) {
         onEvent(event);
       }
-      session.processing = false;
+      processing = false;
       session.abortController = null;
       session.currentRequestId = undefined;
     },

package/src/__tests__/web-search-backend-failure.test.ts

@@ -0,0 +1,166 @@
+// End-to-end integration coverage for the centralized web_search
+// backend-failure normalization layer (ATL-727).
+//
+// PRs 1-4 built the layer in three places:
+//   - `tools/network/web-search-error.ts` — `classifyWebSearchFailure` +
+//     `WEB_SEARCH_BACKEND_FAILURE_MESSAGE` + the structured
+//     `web_search_backend_failure` log helper.
+//   - `daemon/conversation-agent-loop-handlers.ts` — the native
+//     `server_tool_complete` handler maps Anthropic backend failures to the
+//     friendly copy, dedups per turn (`webSearchBackendFailureNotified`), and
+//     gates telemetry on `classification.isBackendFailure`.
+//   - `tools/network/web-search.ts` — the app-side `backendFailureResult`
+//     helper routes 5xx/network/429 to the same copy.
+//
+// The single-layer native-handler invariants (friendly copy + isError + empty
+// results, raw-detail-logged-not-shown, per-turn dedup, successful-empty
+// search) are owned by `native-web-search.test.ts`. This file locks the
+// cross-cutting acceptance criteria that file does not cover:
+//   - honest continuation (the failure is a recoverable tool_result, not a
+//     thrown provider error; the search is never marked successful),
+//   - web_fetch DNS failures are NOT conflated with the search backend copy.
+//
+// It genuinely reuses the shared handler harness in
+// `helpers/native-web-search-harness.ts` (the same harness driven by
+// `native-web-search.test.ts`); the web_fetch invariant is exercised through
+// the same `handleToolResult` path an app-executed fetch tool drives.
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
+
+// ---------------------------------------------------------------------------
+// Mock the daemon collaborators the handler module imports at load time so the
+// handler can be driven in isolation (mirrors native-web-search.test.ts).
+// `mock.module()` is file-scoped, so the shared harness cannot install these.
+// ---------------------------------------------------------------------------
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
+}));
+
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// Import after mocking.
+import {
+  createEventHandlerState,
+  dispatchAgentEvent,
+  type EventHandlerState,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import { WEB_SEARCH_BACKEND_FAILURE_MESSAGE } from "../tools/network/web-search-error.js";
+import {
+  backendFailureLogs,
+  completeNativeWebSearch,
+  createHandlerDeps,
+  lastToolResult,
+} from "./helpers/native-web-search-harness.js";
+
+describe("web_search backend-failure end-to-end (ATL-727)", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("backend failure stays an honest, recoverable tool result (not a thrown provider error, never marked successful)", async () => {
+    const { deps, events } = createHandlerDeps("req-honesty");
+
+    // A recoverable backend failure flows as a tool_result, so dispatch must
+    // resolve without throwing.
+    await expect(
+      completeNativeWebSearch(state, deps, "tu_honesty", {
+        isError: true,
+        errorCode: "unavailable",
+      }),
+    ).resolves.toBeUndefined();
+
+    const result = lastToolResult(events);
+    // The search is never silently upgraded to a success.
+    expect(result?.isError).toBe(true);
+    expect(result?.activityMetadata?.webSearch?.resultCount).toBe(0);
+    expect(result?.activityMetadata?.webSearch?.results).toEqual([]);
+  });
+
+  test("web_fetch DNS failure is NOT conflated with the web_search backend copy", async () => {
+    // The normalization layer keys exclusively on web_search (the native
+    // `server_tool_complete` handler and `web-search.ts`). It never inspects
+    // `webFetch` metadata. handleToolResult forwards an app-executed tool's
+    // `activityMetadata` verbatim, so a web_fetch DNS failure must reach the
+    // client with its own copy and acquire NO `webSearch` metadata. Drive that
+    // path directly with a DNS-failure fetch result for grimgoods.io.
+    const { deps, events, warnings } = createHandlerDeps("req-web-fetch");
+
+    const dnsError =
+      'Error: Unable to resolve host "grimgoods.io" (DNS lookup failed)';
+    const fetchMetadata: ToolActivityMetadata = {
+      webFetch: {
+        url: "https://grimgoods.io",
+        finalUrl: "https://grimgoods.io",
+        status: 0,
+        byteCount: 0,
+        charCount: 0,
+        truncated: false,
+        domain: "grimgoods.io",
+        redirectCount: 0,
+        durationMs: 12,
+        errorMessage: dnsError,
+      },
+    };
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_fetch",
+      name: "web_fetch",
+      input: { url: "https://grimgoods.io" },
+    });
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_result",
+      toolUseId: "tu_fetch",
+      content: dnsError,
+      isError: true,
+      activityMetadata: fetchMetadata,
+    });
+
+    const result = lastToolResult(events);
+    expect(result?.isError).toBe(true);
+
+    // The DNS error keeps its own webFetch copy untouched...
+    expect(result?.activityMetadata?.webFetch?.errorMessage).toBe(dnsError);
+    // ...and is never rewritten to the search backend copy.
+    expect(result?.activityMetadata?.webFetch?.errorMessage).not.toBe(
+      WEB_SEARCH_BACKEND_FAILURE_MESSAGE,
+    );
+    // No webSearch metadata is fabricated for a fetch failure.
+    expect(result?.activityMetadata?.webSearch).toBeUndefined();
+    // And the web_search backend-failure telemetry never fires for a fetch.
+    expect(backendFailureLogs(warnings)).toHaveLength(0);
+  });
+});