@vellumai/assistant 0.8.1 → 0.8.3

package/src/runtime/routes/__tests__/consolidation-routes.test.ts

@@ -0,0 +1,258 @@
+/**
+ * Asserts `listConsolidationRuns` maps background-conversation rows tagged
+ * with `source = MEMORY_V2_CONSOLIDATION_SOURCE` into the heartbeat-runs
+ * response shape, derives `status` / `finishedAt` / `durationMs` from
+ * **assistant-message presence** (not `lastMessageAt`), and clamps the
+ * `limit` query param.
+ *
+ * Synthetic-field semantics covered here:
+ *   - `id` and `conversationId` both equal the conversation row's id.
+ *   - `scheduledFor` and `startedAt` both equal `conversation.createdAt`
+ *     (no separate schedule timestamp on the row).
+ *   - `finishedAt` is the `createdAt` of the LATEST assistant message,
+ *     NOT `conversation.lastMessageAt` — the kickoff user prompt bumps
+ *     `lastMessageAt` before the agent runs, so it cannot be used as a
+ *     completion signal.
+ *   - `durationMs` is `finishedAt − startedAt` when both are present, else
+ *     null.
+ *   - `status` is `"ok"` when the conversation has at least one assistant
+ *     message (positive evidence the agent emitted output) and `"running"`
+ *     otherwise — including the case where only the kickoff user prompt
+ *     has been persisted.
+ *   - `skipReason` and `error` are always null — the conversation row
+ *     alone cannot distinguish a clean run from a mid-flight crash even
+ *     once assistant output exists.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+import { createConversation } from "../../../memory/conversation-crud.js";
+import { getDb } from "../../../memory/db-connection.js";
+import { initializeDb } from "../../../memory/db-init.js";
+import { rawRun } from "../../../memory/raw-query.js";
+import { ROUTES } from "../consolidation-routes.js";
+import type { RouteDefinition } from "../types.js";
+
+initializeDb();
+
+function resetTables(): void {
+  const db = getDb();
+  db.run(`DELETE FROM messages`);
+  db.run(`DELETE FROM conversations`);
+}
+
+function findHandler(operationId: string): RouteDefinition["handler"] {
+  const route = ROUTES.find((r) => r.operationId === operationId);
+  if (!route) throw new Error(`Route ${operationId} not found`);
+  return route.handler;
+}
+
+function insertMessage(
+  conversationId: string,
+  role: string,
+  createdAt: number,
+): void {
+  rawRun(
+    "INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES (?, ?, ?, ?, ?)",
+    `msg-${conversationId}-${role}-${createdAt}`,
+    conversationId,
+    role,
+    "x",
+    createdAt,
+  );
+}
+
+interface RunRecord {
+  id: string;
+  scheduledFor: number;
+  startedAt: number | null;
+  finishedAt: number | null;
+  durationMs: number | null;
+  status: "ok" | "running";
+  skipReason: string | null;
+  error: string | null;
+  conversationId: string | null;
+  createdAt: number;
+}
+
+interface ListRunsResponse {
+  runs: RunRecord[];
+}
+
+describe("listConsolidationRuns handler", () => {
+  beforeEach(() => {
+    resetTables();
+  });
+
+  test("returns only conversations sourced from memory_v2_consolidation", async () => {
+    createConversation({ title: "c1", source: "memory_v2_consolidation" });
+    createConversation({ title: "h1", source: "heartbeat" });
+    createConversation({ title: "u1", source: "user" });
+
+    const handler = findHandler("listConsolidationRuns");
+    const result = (await handler({})) as ListRunsResponse;
+
+    expect(result.runs).toHaveLength(1);
+  });
+
+  test("synthesizes status='ok' with finishedAt from latest assistant message", async () => {
+    const conv = createConversation({
+      title: "c1",
+      source: "memory_v2_consolidation",
+    });
+    rawRun(
+      "UPDATE conversations SET created_at = ? WHERE id = ?",
+      1000,
+      conv.id,
+    );
+    // Kickoff user prompt at t=1100 (bumps lastMessageAt — must NOT be
+    // mistaken for completion).
+    insertMessage(conv.id, "user", 1100);
+    // Agent's first assistant turn at t=2000.
+    insertMessage(conv.id, "assistant", 2000);
+    // Agent's final assistant turn at t=2500.
+    insertMessage(conv.id, "assistant", 2500);
+
+    const handler = findHandler("listConsolidationRuns");
+    const result = (await handler({})) as ListRunsResponse;
+
+    expect(result.runs).toHaveLength(1);
+    const run = result.runs[0]!;
+    expect(run.id).toBe(conv.id);
+    expect(run.conversationId).toBe(conv.id);
+    expect(run.status).toBe("ok");
+    expect(run.scheduledFor).toBe(1000);
+    expect(run.startedAt).toBe(1000);
+    // finishedAt = createdAt of LATEST assistant message (2500), NOT
+    // the conversation's lastMessageAt (which sqlite triggers may or may
+    // not have updated here — irrelevant to this endpoint).
+    expect(run.finishedAt).toBe(2500);
+    expect(run.durationMs).toBe(1500);
+    expect(run.createdAt).toBe(1000);
+  });
+
+  test("synthesizes status='running' when conversation has no assistant message", async () => {
+    createConversation({ title: "c1", source: "memory_v2_consolidation" });
+
+    const handler = findHandler("listConsolidationRuns");
+    const result = (await handler({})) as ListRunsResponse;
+
+    expect(result.runs).toHaveLength(1);
+    const run = result.runs[0]!;
+    expect(run.status).toBe("running");
+    expect(run.finishedAt).toBeNull();
+    expect(run.durationMs).toBeNull();
+  });
+
+  test("status stays 'running' when only the kickoff user prompt exists (Codex bug regression guard)", async () => {
+    // Regression guard for the original `status from lastMessageAt`
+    // heuristic. `processMessage` persists the background kickoff prompt as
+    // a user message BEFORE the agent runs, which bumps
+    // `conversation.lastMessageAt`. A run that timed out / threw before
+    // emitting any assistant turn must still report status='running' (or
+    // an explicit failure status once one exists) — never 'ok'.
+    const conv = createConversation({
+      title: "c1",
+      source: "memory_v2_consolidation",
+    });
+    rawRun(
+      "UPDATE conversations SET created_at = ?, last_message_at = ? WHERE id = ?",
+      1000,
+      1100,
+      conv.id,
+    );
+    insertMessage(conv.id, "user", 1100);
+
+    const handler = findHandler("listConsolidationRuns");
+    const result = (await handler({})) as ListRunsResponse;
+
+    expect(result.runs).toHaveLength(1);
+    const run = result.runs[0]!;
+    expect(run.status).toBe("running");
+    expect(run.finishedAt).toBeNull();
+    expect(run.durationMs).toBeNull();
+  });
+
+  test("skipReason and error are always null (not derivable from conversation row)", async () => {
+    const conv = createConversation({
+      title: "c1",
+      source: "memory_v2_consolidation",
+    });
+    insertMessage(conv.id, "assistant", 2000);
+
+    const handler = findHandler("listConsolidationRuns");
+    const result = (await handler({})) as ListRunsResponse;
+
+    expect(result.runs[0]!.skipReason).toBeNull();
+    expect(result.runs[0]!.error).toBeNull();
+  });
+
+  test("orders runs by createdAt descending", async () => {
+    const a = createConversation({
+      title: "a",
+      source: "memory_v2_consolidation",
+    });
+    const b = createConversation({
+      title: "b",
+      source: "memory_v2_consolidation",
+    });
+    const c = createConversation({
+      title: "c",
+      source: "memory_v2_consolidation",
+    });
+    rawRun("UPDATE conversations SET created_at = ? WHERE id = ?", 1000, a.id);
+    rawRun("UPDATE conversations SET created_at = ? WHERE id = ?", 3000, b.id);
+    rawRun("UPDATE conversations SET created_at = ? WHERE id = ?", 2000, c.id);
+
+    const handler = findHandler("listConsolidationRuns");
+    const result = (await handler({})) as ListRunsResponse;
+
+    expect(result.runs.map((r) => r.id)).toEqual([b.id, c.id, a.id]);
+  });
+
+  test("limit defaults to 20, clamps to [1, 100], and falls back on non-numeric input", async () => {
+    for (let i = 0; i < 5; i++) {
+      createConversation({
+        title: `c${i}`,
+        source: "memory_v2_consolidation",
+      });
+    }
+
+    const handler = findHandler("listConsolidationRuns");
+
+    // Default — all 5 returned (under the 20 default).
+    const def = (await handler({})) as ListRunsResponse;
+    expect(def.runs).toHaveLength(5);
+
+    // Explicit limit honored.
+    const lim2 = (await handler({
+      queryParams: { limit: "2" },
+    })) as ListRunsResponse;
+    expect(lim2.runs).toHaveLength(2);
+
+    // Negative clamps to 1.
+    const neg = (await handler({
+      queryParams: { limit: "-5" },
+    })) as ListRunsResponse;
+    expect(neg.runs).toHaveLength(1);
+
+    // Zero clamps to 1.
+    const zero = (await handler({
+      queryParams: { limit: "0" },
+    })) as ListRunsResponse;
+    expect(zero.runs).toHaveLength(1);
+
+    // Non-numeric falls back to the default (20 → all 5 here).
+    const bad = (await handler({
+      queryParams: { limit: "garbage" },
+    })) as ListRunsResponse;
+    expect(bad.runs).toHaveLength(5);
+  });
+});

package/src/runtime/routes/__tests__/conversation-management-routes.test.ts

@@ -23,6 +23,7 @@ mock.module("../../assistant-event-hub.js", () => ({
     publish: async () => {},
     subscribe: () => () => {},
   },
+  broadcastMessage: () => {},
 }));
 
 import { getDb } from "../../../memory/db-connection.js";
@@ -114,7 +115,10 @@ function seedConversation(id: string): void {
 describe("PUT /v1/conversations/:id/inference-profile", () => {
   beforeEach(() => {
     clearConversations();
-    configLlmProfiles = { fast: { model: "model-a" }, slow: { model: "model-b" } };
+    configLlmProfiles = {
+      fast: { model: "model-a" },
+      slow: { model: "model-b" },
+    };
   });
 
   test("PUT with ttlSeconds=600 → response includes sessionId (UUID), expiresAt, ttlSeconds=600", async () => {

package/src/runtime/routes/__tests__/conversation-query-routes.test.ts

@@ -14,6 +14,12 @@ import {
 
 let rawConfigFixture: Record<string, unknown> = {};
 let savedRawConfig: Record<string, unknown> | null = null;
+// Counters / spies so tests can assert that `commitConfigWrite` ran its
+// post-write side effects. Each `replaceProfileRoute.handler` call that
+// hits `commitConfigWrite` should bump these once.
+let invalidateConfigCacheCalls = 0;
+let initializeProvidersCalls = 0;
+let clearEmbeddingBackendCacheCalls = 0;
 
 mock.module("../../../config/loader.js", () => ({
   loadRawConfig: () => structuredClone(rawConfigFixture),
@@ -26,6 +32,28 @@ mock.module("../../../config/loader.js", () => ({
   ) => {
     Object.assign(target, overrides);
   },
+  // `commitConfigWrite` (used by `handleReplaceInferenceProfile`) pulls
+  // in `getConfig` for the provider reinit's config arg and
+  // `invalidateConfigCache` so the next caller sees the fresh write.
+  // Stub both: getConfig returns whatever was last saved (or the fixture
+  // if nothing has been saved yet) and the cache-invalidation function
+  // is a counter so we can assert it fired.
+  getConfig: () => structuredClone(savedRawConfig ?? rawConfigFixture),
+  invalidateConfigCache: () => {
+    invalidateConfigCacheCalls += 1;
+  },
+}));
+
+mock.module("../../../providers/registry.js", () => ({
+  initializeProviders: async () => {
+    initializeProvidersCalls += 1;
+  },
+}));
+
+mock.module("../../../memory/embedding-backend.js", () => ({
+  clearEmbeddingBackendCache: () => {
+    clearEmbeddingBackendCacheCalls += 1;
+  },
 }));
 
 import type { ConversationCreateType } from "../../../memory/conversation-crud.js";
@@ -43,6 +71,10 @@ import {
   memoryV2ActivationLogs,
   messages,
 } from "../../../memory/schema.js";
+import {
+  createConnection,
+  getConnection,
+} from "../../../providers/inference/connections.js";
 import { ROUTES } from "../conversation-query-routes.js";
 
 // Local subset: this test only exercises a single concept row.
@@ -288,6 +320,9 @@ describe("GET /v1/messages/:id/llm-context — conversationTotalEstimatedCostUsd
 describe("PUT /v1/config/llm/profiles/:name", () => {
   beforeEach(() => {
     savedRawConfig = null;
+    invalidateConfigCacheCalls = 0;
+    initializeProvidersCalls = 0;
+    clearEmbeddingBackendCacheCalls = 0;
     rawConfigFixture = {
       llm: {
         profiles: {
@@ -313,8 +348,8 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
     };
   });
 
-  test("owns contextWindow maxInputTokens while preserving non-UI profile leaves", () => {
-    const result = replaceProfileRoute.handler({
+  test("owns contextWindow maxInputTokens while preserving non-UI profile leaves", async () => {
+    const result = await replaceProfileRoute.handler({
       pathParams: { name: "custom" },
       body: {
         provider: "openai",
@@ -343,8 +378,8 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
     expect(savedProfile.openrouter).toEqual({ only: ["anthropic"] });
   });
 
-  test("writes only the replacement contextWindow maxInputTokens override", () => {
-    const result = replaceProfileRoute.handler({
+  test("writes only the replacement contextWindow maxInputTokens override", async () => {
+    const result = await replaceProfileRoute.handler({
       pathParams: { name: "custom" },
       body: {
         provider: "openai",
@@ -375,8 +410,8 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
     expect(savedProfile.openrouter).toEqual({ only: ["anthropic"] });
   });
 
-  test("writes provider_connection when present in body", () => {
-    const result = replaceProfileRoute.handler({
+  test("writes provider_connection when present in body", async () => {
+    const result = await replaceProfileRoute.handler({
       pathParams: { name: "custom" },
       body: {
         provider: "openai",
@@ -396,7 +431,7 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
     expect(savedProfile.provider_connection).toBe("personal-openai");
   });
 
-  test("clears provider_connection when omitted from body (UI-owned key)", () => {
+  test("auto-derives provider_connection when omitted from body (Any active)", async () => {
     // Seed an existing binding so the test starts from a non-empty state.
     (
       rawConfigFixture.llm as {
@@ -404,14 +439,63 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
       }
     ).profiles.custom.provider_connection = "stale-openai";
 
-    const result = replaceProfileRoute.handler({
+    const result = await replaceProfileRoute.handler({
       pathParams: { name: "custom" },
       body: {
         provider: "openai",
         model: "gpt-5.5",
         // provider_connection deliberately omitted — the UI cleared the
-        // picker back to "Any active" and the route must wipe the saved
-        // binding, not silently round-trip it.
+        // picker back to "Any active". The route auto-derives an active
+        // connection for the provider to prevent stale inheritance during
+        // config deep-merge.
+      },
+    });
+
+    expect(result).toEqual({ ok: true });
+    const savedProfile = (
+      savedRawConfig?.llm as {
+        profiles: Record<string, Record<string, unknown>>;
+      }
+    ).profiles.custom;
+
+    // The canonical "openai-managed" connection exists in the test DB;
+    // the route auto-derives it when the UI omits provider_connection.
+    expect(savedProfile.provider_connection).toBe("openai-managed");
+  });
+
+  test("auto-derives provider_connection for BYOK provider (Any active)", async () => {
+    // Seed a fireworks connection in the DB.
+    createConnection(getDb(), {
+      name: "fireworks",
+      provider: "fireworks",
+      auth: { type: "api_key", credential: "fireworks:api_key" },
+    });
+
+    const result = await replaceProfileRoute.handler({
+      pathParams: { name: "custom" },
+      body: {
+        provider: "fireworks",
+        model: "accounts/fireworks/models/llama-v3p1-8b-instruct",
+      },
+    });
+
+    expect(result).toEqual({ ok: true });
+    const savedProfile = (
+      savedRawConfig?.llm as {
+        profiles: Record<string, Record<string, unknown>>;
+      }
+    ).profiles.custom;
+
+    expect(savedProfile.provider).toBe("fireworks");
+    expect(savedProfile.provider_connection).toBe("fireworks");
+  });
+
+  test("auto-creates provider_connection when no connection exists for provider", async () => {
+    const result = await replaceProfileRoute.handler({
+      pathParams: { name: "custom" },
+      body: {
+        provider: "openrouter",
+        model: "anthropic/claude-sonnet-4-6",
       },
     });
 
@@ -422,7 +506,16 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
       }
     ).profiles.custom;
 
-    expect(savedProfile.provider_connection).toBeUndefined();
+    expect(savedProfile.provider).toBe("openrouter");
+    expect(savedProfile.provider_connection).toBe("openrouter-personal");
+
+    const conn = getConnection(getDb(), "openrouter-personal");
+    expect(conn).not.toBeNull();
+    expect(conn!.provider).toBe("openrouter");
+    expect(conn!.auth).toEqual({
+      type: "api_key",
+      credential: "credential/openrouter/api_key",
+    });
   });
 
   describe("managed profile guard", () => {
@@ -439,8 +532,8 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
       };
     });
 
-    test("allows label edit on managed profile, preserving seed fields", () => {
-      const result = replaceProfileRoute.handler({
+    test("allows label edit on managed profile, preserving seed fields", async () => {
+      const result = await replaceProfileRoute.handler({
         pathParams: { name: "balanced" },
         body: { label: "My Balanced" },
       });
@@ -459,8 +552,8 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
       expect(savedProfile.source).toBe("managed");
     });
 
-    test("allows status edit on managed profile", () => {
-      const result = replaceProfileRoute.handler({
+    test("allows status edit on managed profile", async () => {
+      const result = await replaceProfileRoute.handler({
         pathParams: { name: "balanced" },
         body: { status: "disabled" },
       });
@@ -476,8 +569,8 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
       expect(savedProfile.provider).toBe("anthropic");
     });
 
-    test("allows label+status edit together", () => {
-      const result = replaceProfileRoute.handler({
+    test("allows label+status edit together", async () => {
+      const result = await replaceProfileRoute.handler({
         pathParams: { name: "balanced" },
         body: { label: "Renamed", status: "disabled" },
       });
@@ -493,25 +586,81 @@ describe("PUT /v1/config/llm/profiles/:name", () => {
       expect(savedProfile.status).toBe("disabled");
     });
 
-    test("rejects provider edit on managed profile with disallowed-keys error", () => {
-      expect(() =>
+    test("rejects provider edit on managed profile with disallowed-keys error", async () => {
+      // The handler is `async`, so synchronous BadRequest throws still
+      // surface as a rejected promise; assert via `.rejects.toThrow`.
+      await expect(
         replaceProfileRoute.handler({
           pathParams: { name: "balanced" },
           body: { provider: "openai", model: "gpt-5" },
         }),
-      ).toThrow(/Cannot edit managed profile "balanced" fields \[provider, model\]/);
+      ).rejects.toThrow(
+        /Cannot edit managed profile "balanced" fields \[provider, model\]/,
+      );
     });
 
-    test("rejects mixed allowed+disallowed fields", () => {
+    test("rejects mixed allowed+disallowed fields", async () => {
       // label is allowed but maxTokens is not — must reject without partially
       // applying label, so saver should never be invoked.
-      expect(() =>
+      await expect(
         replaceProfileRoute.handler({
           pathParams: { name: "balanced" },
           body: { label: "Try", maxTokens: 999 },
         }),
-      ).toThrow(/Cannot edit managed profile "balanced" fields \[maxTokens\]/);
+      ).rejects.toThrow(
+        /Cannot edit managed profile "balanced" fields \[maxTokens\]/,
+      );
       expect(savedRawConfig).toBeNull();
+      // Reject path skips commitConfigWrite entirely — no provider reinit
+      // or cache invalidation should fire on a guard rejection.
+      expect(initializeProvidersCalls).toBe(0);
+      expect(invalidateConfigCacheCalls).toBe(0);
+      expect(clearEmbeddingBackendCacheCalls).toBe(0);
+    });
+  });
+
+  describe("commitConfigWrite side effects", () => {
+    test("status flip on managed profile triggers provider reinit + cache invalidation", async () => {
+      // Seed a managed profile that the user will disable. commitConfigWrite
+      // must reinit the provider registry so the status change is reflected
+      // in the running daemon immediately, not at the next watcher tick.
+      (rawConfigFixture.llm as { profiles: Record<string, unknown> }).profiles[
+        "balanced"
+      ] = {
+        source: "managed",
+        provider: "anthropic",
+        model: "claude-sonnet-4-6",
+        label: "Balanced",
+        status: "active",
+      };
+
+      const result = await replaceProfileRoute.handler({
+        pathParams: { name: "balanced" },
+        body: { status: "disabled" },
+      });
+
+      expect(result).toEqual({ ok: true });
+      expect(initializeProvidersCalls).toBe(1);
+      expect(invalidateConfigCacheCalls).toBe(1);
+      expect(clearEmbeddingBackendCacheCalls).toBe(1);
+    });
+
+    test("custom profile provider swap triggers provider reinit + cache invalidation", async () => {
+      // Custom profile path: provider/model swap on a user-owned profile.
+      // Same side-effect contract — registry must reinit so the new
+      // provider is wired into the running daemon without restart.
+      const result = await replaceProfileRoute.handler({
+        pathParams: { name: "custom" },
+        body: {
+          provider: "openai",
+          model: "gpt-5.5",
+        },
+      });
+
+      expect(result).toEqual({ ok: true });
+      expect(initializeProvidersCalls).toBe(1);
+      expect(invalidateConfigCacheCalls).toBe(1);
+      expect(clearEmbeddingBackendCacheCalls).toBe(1);
     });
   });
 });