@vellumai/assistant 0.4.16 → 0.4.18

package/src/__tests__/memory-retrieval-budget.test.ts

@@ -1,9 +1,9 @@
-import { describe, expect, test } from 'bun:test';
+import { describe, expect, test } from "bun:test";
 
-import { computeRecallBudget } from '../memory/retrieval-budget.js';
+import { computeRecallBudget } from "../memory/retrieval-budget.js";
 
-describe('memory retrieval budget', () => {
-  test('clamps to maxInjectTokens when headroom is large', () => {
+describe("memory retrieval budget", () => {
+  test("clamps to maxInjectTokens when headroom is large", () => {
     const budget = computeRecallBudget({
       estimatedPromptTokens: 20_000,
       maxInputTokens: 180_000,
@@ -14,7 +14,7 @@ describe('memory retrieval budget', () => {
     expect(budget).toBe(10_000);
   });
 
-  test('clamps to minInjectTokens when headroom is tight', () => {
+  test("clamps to minInjectTokens when headroom is tight", () => {
     const budget = computeRecallBudget({
       estimatedPromptTokens: 172_000,
       maxInputTokens: 180_000,
@@ -25,7 +25,7 @@ describe('memory retrieval budget', () => {
     expect(budget).toBe(1_200);
   });
 
-  test('returns computed value when between min and max', () => {
+  test("returns computed value when between min and max", () => {
     const budget = computeRecallBudget({
       estimatedPromptTokens: 165_000,
       maxInputTokens: 180_000,
@@ -36,7 +36,7 @@ describe('memory retrieval budget', () => {
     expect(budget).toBe(7_000);
   });
 
-  test('normalizes invalid min/max ordering safely', () => {
+  test("normalizes invalid min/max ordering safely", () => {
     const budget = computeRecallBudget({
       estimatedPromptTokens: 150_000,
       maxInputTokens: 180_000,
@@ -47,4 +47,3 @@ describe('memory retrieval budget', () => {
     expect(budget).toBe(12_000);
   });
 });
-

package/src/__tests__/memory-retrieval.benchmark.test.ts

@@ -5,37 +5,45 @@
  * Validates latency stays within acceptable bounds and token budget
  * enforcement works correctly.
  */
-import { mkdtempSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-
-import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from 'bun:test';
-
-const testDir = mkdtempSync(join(tmpdir(), 'mem-retrieval-bench-'));
-
-mock.module('../util/platform.js', () => ({
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  mock,
+  test,
+} from "bun:test";
+
+const testDir = mkdtempSync(join(tmpdir(), "mem-retrieval-bench-"));
+
+mock.module("../util/platform.js", () => ({
   getDataDir: () => testDir,
-  isMacOS: () => process.platform === 'darwin',
-  isLinux: () => process.platform === 'linux',
-  isWindows: () => process.platform === 'win32',
-  getSocketPath: () => join(testDir, 'test.sock'),
-  getPidPath: () => join(testDir, 'test.pid'),
-  getDbPath: () => join(testDir, 'test.db'),
-  getLogPath: () => join(testDir, 'test.log'),
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getSocketPath: () => join(testDir, "test.sock"),
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
   ensureDataDir: () => {},
 }));
 
-mock.module('../util/logger.js', () => ({
-  getLogger: () => new Proxy({} as Record<string, unknown>, {
-    get: () => () => {},
-  }),
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
 }));
 
 // Counter for semantic search invocations — used to verify early termination
 // skips the call entirely rather than relying on flaky wall-clock comparisons.
 let semanticSearchCallCount = 0;
 
-mock.module('../memory/search/semantic.js', () => ({
+mock.module("../memory/search/semantic.js", () => ({
   semanticSearch: async () => {
     semanticSearchCallCount++;
     return [];
@@ -43,64 +51,76 @@ mock.module('../memory/search/semantic.js', () => ({
   isQdrantConnectionError: () => false,
 }));
 
-mock.module('../memory/embedding-backend.js', () => ({
+mock.module("../memory/embedding-backend.js", () => ({
   getMemoryBackendStatus: (config: { memory: { enabled: boolean } }) => ({
     enabled: config.memory.enabled,
     degraded: false,
-    provider: 'local',
-    model: 'mock-embedding',
+    provider: "local",
+    model: "mock-embedding",
     reason: null,
   }),
   embedWithBackend: async () => ({
-    provider: 'local' as const,
-    model: 'mock-embedding',
+    provider: "local" as const,
+    model: "mock-embedding",
     vectors: [new Array(1536).fill(0)],
   }),
 }));
 
-import { DEFAULT_CONFIG } from '../config/defaults.js';
-import type { AssistantConfig } from '../config/types.js';
-import { getDb, initializeDb, resetDb } from '../memory/db.js';
-import { buildMemoryRecall } from '../memory/retriever.js';
-import { conversations, memorySegments, messages } from '../memory/schema.js';
-
-function seedMemoryItems(conversationId: string, count: number, now: number): void {
+import { DEFAULT_CONFIG } from "../config/defaults.js";
+import type { AssistantConfig } from "../config/types.js";
+import { getDb, initializeDb, resetDb } from "../memory/db.js";
+import { buildMemoryRecall } from "../memory/retriever.js";
+import { conversations, memorySegments, messages } from "../memory/schema.js";
+
+function seedMemoryItems(
+  conversationId: string,
+  count: number,
+  now: number,
+): void {
   const db = getDb();
-  db.insert(conversations).values({
-    id: conversationId,
-    title: null,
-    createdAt: now,
-    updatedAt: now,
-    totalInputTokens: 0,
-    totalOutputTokens: 0,
-    totalEstimatedCost: 0,
-    contextSummary: null,
-    contextCompactedMessageCount: 0,
-    contextCompactedAt: null,
-  }).run();
+  db.insert(conversations)
+    .values({
+      id: conversationId,
+      title: null,
+      createdAt: now,
+      updatedAt: now,
+      totalInputTokens: 0,
+      totalOutputTokens: 0,
+      totalEstimatedCost: 0,
+      contextSummary: null,
+      contextCompactedMessageCount: 0,
+      contextCompactedAt: null,
+    })
+    .run();
 
   for (let i = 0; i < count; i++) {
     const msgId = `msg-${conversationId}-${i}`;
-    const text = `Memory item ${i}: information about topic-${i % 20} including keyword-${i % 10} details.`;
-    db.insert(messages).values({
-      id: msgId,
-      conversationId,
-      role: i % 2 === 0 ? 'user' : 'assistant',
-      content: JSON.stringify([{ type: 'text', text }]),
-      createdAt: now + i,
-    }).run();
-    db.insert(memorySegments).values({
-      id: `seg-${conversationId}-${i}`,
-      messageId: msgId,
-      conversationId,
-      role: i % 2 === 0 ? 'user' : 'assistant',
-      segmentIndex: 0,
-      text,
-      tokenEstimate: 20,
-      scopeId: 'default',
-      createdAt: now + i,
-      updatedAt: now + i,
-    }).run();
+    const text = `Memory item ${i}: information about topic-${
+      i % 20
+    } including keyword-${i % 10} details.`;
+    db.insert(messages)
+      .values({
+        id: msgId,
+        conversationId,
+        role: i % 2 === 0 ? "user" : "assistant",
+        content: JSON.stringify([{ type: "text", text }]),
+        createdAt: now + i,
+      })
+      .run();
+    db.insert(memorySegments)
+      .values({
+        id: `seg-${conversationId}-${i}`,
+        messageId: msgId,
+        conversationId,
+        role: i % 2 === 0 ? "user" : "assistant",
+        segmentIndex: 0,
+        text,
+        tokenEstimate: 20,
+        scopeId: "default",
+        createdAt: now + i,
+        updatedAt: now + i,
+      })
+      .run();
   }
 }
 
@@ -111,7 +131,7 @@ function makeConfig(overrides?: { maxInjectTokens?: number }): AssistantConfig {
       ...DEFAULT_CONFIG.memory,
       embeddings: {
         ...DEFAULT_CONFIG.memory.embeddings,
-        provider: 'local' as const,
+        provider: "local" as const,
         required: false,
       },
       retrieval: {
@@ -119,7 +139,10 @@ function makeConfig(overrides?: { maxInjectTokens?: number }): AssistantConfig {
         lexicalTopK: 50,
         semanticTopK: 20,
         maxInjectTokens: overrides?.maxInjectTokens ?? 750,
-        reranking: { ...DEFAULT_CONFIG.memory.retrieval.reranking, enabled: false },
+        reranking: {
+          ...DEFAULT_CONFIG.memory.retrieval.reranking,
+          enabled: false,
+        },
         dynamicBudget: {
           enabled: false,
           minInjectTokens: 160,
@@ -131,26 +154,26 @@ function makeConfig(overrides?: { maxInjectTokens?: number }): AssistantConfig {
   };
 }
 
-describe('Memory retrieval benchmark', () => {
+describe("Memory retrieval benchmark", () => {
   beforeAll(() => {
     initializeDb();
   });
 
   beforeEach(() => {
     const db = getDb();
-    db.run('DELETE FROM memory_item_sources');
-    db.run('DELETE FROM memory_item_entities');
-    db.run('DELETE FROM memory_entity_relations');
-    db.run('DELETE FROM memory_entities');
-    db.run('DELETE FROM memory_embeddings');
-    db.run('DELETE FROM memory_summaries');
-    db.run('DELETE FROM memory_items');
-    db.run('DELETE FROM memory_segment_fts');
-    db.run('DELETE FROM memory_segments');
-    db.run('DELETE FROM messages');
-    db.run('DELETE FROM conversations');
-    db.run('DELETE FROM memory_jobs');
-    db.run('DELETE FROM memory_checkpoints');
+    db.run("DELETE FROM memory_item_sources");
+    db.run("DELETE FROM memory_item_entities");
+    db.run("DELETE FROM memory_entity_relations");
+    db.run("DELETE FROM memory_entities");
+    db.run("DELETE FROM memory_embeddings");
+    db.run("DELETE FROM memory_summaries");
+    db.run("DELETE FROM memory_items");
+    db.run("DELETE FROM memory_segment_fts");
+    db.run("DELETE FROM memory_segments");
+    db.run("DELETE FROM messages");
+    db.run("DELETE FROM conversations");
+    db.run("DELETE FROM memory_jobs");
+    db.run("DELETE FROM memory_checkpoints");
   });
 
   afterAll(() => {
@@ -162,14 +185,14 @@ describe('Memory retrieval benchmark', () => {
     }
   });
 
-  test('retrieval completes under 500ms for 100 items', async () => {
-    const conversationId = 'conv-bench-100';
+  test("retrieval completes under 500ms for 100 items", async () => {
+    const conversationId = "conv-bench-100";
     const now = 1_700_500_000_000;
     seedMemoryItems(conversationId, 100, now);
 
     const config = makeConfig();
     const recall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       config,
     );
@@ -182,14 +205,14 @@ describe('Memory retrieval benchmark', () => {
     expect(recall.latencyMs).toBeLessThan(500);
   });
 
-  test('retrieval completes under 1000ms for 500 items', async () => {
-    const conversationId = 'conv-bench-500';
+  test("retrieval completes under 1000ms for 500 items", async () => {
+    const conversationId = "conv-bench-500";
     const now = 1_700_500_000_000;
     seedMemoryItems(conversationId, 500, now);
 
     const config = makeConfig();
     const recall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       config,
     );
@@ -201,14 +224,14 @@ describe('Memory retrieval benchmark', () => {
     expect(recall.latencyMs).toBeLessThan(1000);
   });
 
-  test('retrieval completes under 2000ms for 2000 items', async () => {
-    const conversationId = 'conv-bench-2000';
+  test("retrieval completes under 2000ms for 2000 items", async () => {
+    const conversationId = "conv-bench-2000";
     const now = 1_700_500_000_000;
     seedMemoryItems(conversationId, 2000, now);
 
     const config = makeConfig();
     const recall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       config,
     );
@@ -220,15 +243,15 @@ describe('Memory retrieval benchmark', () => {
     expect(recall.latencyMs).toBeLessThan(2000);
   });
 
-  test('token budget enforcement: maxInjectTokens is respected', async () => {
-    const conversationId = 'conv-bench-budget';
+  test("token budget enforcement: maxInjectTokens is respected", async () => {
+    const conversationId = "conv-bench-budget";
     const now = 1_700_500_000_000;
     seedMemoryItems(conversationId, 500, now);
 
     const smallBudget = 200;
     const config = makeConfig({ maxInjectTokens: smallBudget });
     const recall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       config,
     );
@@ -241,18 +264,20 @@ describe('Memory retrieval benchmark', () => {
     const largeBudget = 2000;
     const largeConfig = makeConfig({ maxInjectTokens: largeBudget });
     const largeRecall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       largeConfig,
     );
 
     expect(largeRecall.injectedTokens).toBeLessThanOrEqual(largeBudget);
     // With more budget, we should get at least as many tokens
-    expect(largeRecall.injectedTokens).toBeGreaterThanOrEqual(recall.injectedTokens);
+    expect(largeRecall.injectedTokens).toBeGreaterThanOrEqual(
+      recall.injectedTokens,
+    );
   });
 
-  test('early termination reduces latency when applicable', async () => {
-    const conversationId = 'conv-bench-et';
+  test("early termination reduces latency when applicable", async () => {
+    const conversationId = "conv-bench-et";
     const now = 1_700_500_000_000;
     // Seed enough items that early termination can trigger
     seedMemoryItems(conversationId, 500, now);
@@ -264,7 +289,7 @@ describe('Memory retrieval benchmark', () => {
         ...DEFAULT_CONFIG.memory,
         embeddings: {
           ...DEFAULT_CONFIG.memory.embeddings,
-          provider: 'local' as const,
+          provider: "local" as const,
           required: false,
         },
         retrieval: {
@@ -272,7 +297,10 @@ describe('Memory retrieval benchmark', () => {
           lexicalTopK: 50,
           semanticTopK: 20,
           maxInjectTokens: 750,
-          reranking: { ...DEFAULT_CONFIG.memory.retrieval.reranking, enabled: false },
+          reranking: {
+            ...DEFAULT_CONFIG.memory.retrieval.reranking,
+            enabled: false,
+          },
           dynamicBudget: {
             enabled: false,
             minInjectTokens: 160,
@@ -290,7 +318,7 @@ describe('Memory retrieval benchmark', () => {
     };
 
     const recall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       etConfig,
     );
@@ -302,12 +330,12 @@ describe('Memory retrieval benchmark', () => {
     expect(recall.selectedCount).toBeGreaterThan(0);
   });
 
-  test('early termination skips semantic search entirely', async () => {
-    const conversationId = 'conv-bench-et-skip';
+  test("early termination skips semantic search entirely", async () => {
+    const conversationId = "conv-bench-et-skip";
     const now = 1_700_500_000_000;
     seedMemoryItems(conversationId, 500, now);
 
-    const query = 'What do we know about topic-5 and keyword-3?';
+    const query = "What do we know about topic-5 and keyword-3?";
 
     const etConfig: AssistantConfig = {
       ...DEFAULT_CONFIG,
@@ -315,7 +343,7 @@ describe('Memory retrieval benchmark', () => {
         ...DEFAULT_CONFIG.memory,
         embeddings: {
           ...DEFAULT_CONFIG.memory.embeddings,
-          provider: 'local' as const,
+          provider: "local" as const,
           required: false,
         },
         retrieval: {
@@ -323,7 +351,10 @@ describe('Memory retrieval benchmark', () => {
           lexicalTopK: 50,
           semanticTopK: 20,
           maxInjectTokens: 750,
-          reranking: { ...DEFAULT_CONFIG.memory.retrieval.reranking, enabled: false },
+          reranking: {
+            ...DEFAULT_CONFIG.memory.retrieval.reranking,
+            enabled: false,
+          },
           dynamicBudget: {
             enabled: false,
             minInjectTokens: 160,
@@ -367,15 +398,19 @@ describe('Memory retrieval benchmark', () => {
 
     // Run without ET — semantic search should be invoked
     semanticSearchCallCount = 0;
-    const baselineRecall = await buildMemoryRecall(query, conversationId, noEtConfig);
+    const baselineRecall = await buildMemoryRecall(
+      query,
+      conversationId,
+      noEtConfig,
+    );
     const baselineCalls = semanticSearchCallCount;
 
     expect(baselineRecall.earlyTerminated).toBe(false);
     expect(baselineCalls).toBeGreaterThan(0);
   });
 
-  test('recall.latencyMs tracks wall-clock within 50% tolerance', async () => {
-    const conversationId = 'conv-bench-wallclock';
+  test("recall.latencyMs tracks wall-clock within 50% tolerance", async () => {
+    const conversationId = "conv-bench-wallclock";
     const now = 1_700_500_000_000;
     seedMemoryItems(conversationId, 500, now);
 
@@ -386,7 +421,7 @@ describe('Memory retrieval benchmark', () => {
     // integer-ms Date.now() and sub-ms performance.now().
     const wallStart = Date.now();
     const recall = await buildMemoryRecall(
-      'What do we know about topic-5 and keyword-3?',
+      "What do we know about topic-5 and keyword-3?",
       conversationId,
       config,
     );