@vellumai/assistant 0.5.16 → 0.6.1

package/src/__tests__/memory-retrieval.benchmark.test.ts

@@ -1,295 +0,0 @@
-/**
- * Memory Retrieval Benchmark
- *
- * Measures end-to-end memory recall time with varying database sizes.
- * Validates latency stays within acceptable bounds and token budget
- * enforcement works correctly.
- *
- * The pipeline uses hybrid search (Qdrant) only. With Qdrant mocked empty,
- * no candidates are found and injectedText is empty. The tests verify
- * pipeline completion, latency bounds, and token budget enforcement.
- */
-import {
-  afterAll,
-  beforeAll,
-  beforeEach,
-  describe,
-  expect,
-  mock,
-  test,
-} from "bun:test";
-
-mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
-}));
-
-// Counter for semantic search invocations
-let semanticSearchCallCount = 0;
-
-mock.module("../memory/search/semantic.js", () => ({
-  semanticSearch: async () => {
-    semanticSearchCallCount++;
-    return [];
-  },
-  isQdrantConnectionError: () => false,
-}));
-
-mock.module("../memory/embedding-backend.js", () => ({
-  getMemoryBackendStatus: async (config: { memory: { enabled: boolean } }) => ({
-    enabled: config.memory.enabled,
-    degraded: false,
-    provider: "local",
-    model: "mock-embedding",
-    reason: null,
-  }),
-  embedWithBackend: async () => ({
-    provider: "local" as const,
-    model: "mock-embedding",
-    vectors: [new Array(1536).fill(0)],
-  }),
-  generateSparseEmbedding: () => ({ indices: [], values: [] }),
-  logMemoryEmbeddingWarning: () => {},
-}));
-
-import { DEFAULT_CONFIG } from "../config/defaults.js";
-import type { AssistantConfig } from "../config/types.js";
-import { getDb, initializeDb, resetDb } from "../memory/db.js";
-import { buildMemoryRecall } from "../memory/retriever.js";
-import { conversations, memorySegments, messages } from "../memory/schema.js";
-
-function seedMemoryItems(
-  conversationId: string,
-  count: number,
-  now: number,
-): void {
-  const db = getDb();
-  db.insert(conversations)
-    .values({
-      id: conversationId,
-      title: null,
-      createdAt: now,
-      updatedAt: now,
-      totalInputTokens: 0,
-      totalOutputTokens: 0,
-      totalEstimatedCost: 0,
-      contextSummary: null,
-      contextCompactedMessageCount: 0,
-      contextCompactedAt: null,
-    })
-    .run();
-
-  for (let i = 0; i < count; i++) {
-    const msgId = `msg-${conversationId}-${i}`;
-    const text = `Memory item ${i}: information about topic-${
-      i % 20
-    } including keyword-${i % 10} details.`;
-    db.insert(messages)
-      .values({
-        id: msgId,
-        conversationId,
-        role: i % 2 === 0 ? "user" : "assistant",
-        content: JSON.stringify([{ type: "text", text }]),
-        createdAt: now + i,
-      })
-      .run();
-    db.insert(memorySegments)
-      .values({
-        id: `seg-${conversationId}-${i}`,
-        messageId: msgId,
-        conversationId,
-        role: i % 2 === 0 ? "user" : "assistant",
-        segmentIndex: 0,
-        text,
-        tokenEstimate: 20,
-        scopeId: "default",
-        createdAt: now + i,
-        updatedAt: now + i,
-      })
-      .run();
-  }
-}
-
-function makeConfig(overrides?: { maxInjectTokens?: number }): AssistantConfig {
-  return {
-    ...DEFAULT_CONFIG,
-    memory: {
-      ...DEFAULT_CONFIG.memory,
-      embeddings: {
-        ...DEFAULT_CONFIG.memory.embeddings,
-        provider: "local" as const,
-        required: false,
-      },
-      retrieval: {
-        ...DEFAULT_CONFIG.memory.retrieval,
-        maxInjectTokens: overrides?.maxInjectTokens ?? 750,
-        dynamicBudget: {
-          enabled: false,
-          minInjectTokens: 160,
-          maxInjectTokens: overrides?.maxInjectTokens ?? 750,
-          targetHeadroomTokens: 900,
-        },
-      },
-    },
-  };
-}
-
-describe("Memory retrieval benchmark", () => {
-  beforeAll(() => {
-    initializeDb();
-  });
-
-  beforeEach(() => {
-    const db = getDb();
-    db.run("DELETE FROM memory_item_sources");
-    db.run("DELETE FROM memory_embeddings");
-    db.run("DELETE FROM memory_items");
-
-    db.run("DELETE FROM memory_segments");
-    db.run("DELETE FROM messages");
-    db.run("DELETE FROM conversations");
-    db.run("DELETE FROM memory_jobs");
-    db.run("DELETE FROM memory_checkpoints");
-  });
-
-  afterAll(() => {
-    resetDb();
-  });
-
-  test("retrieval completes under 500ms for 100 items", async () => {
-    const conversationId = "conv-bench-100";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 100, now);
-
-    const config = makeConfig();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    expect(recall.degraded).toBe(false);
-    // Relaxed threshold — guards against severe regressions, not precise benchmarking
-    expect(recall.latencyMs).toBeLessThan(500);
-  });
-
-  test("retrieval completes under 1000ms for 500 items", async () => {
-    const conversationId = "conv-bench-500";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 500, now);
-
-    const config = makeConfig();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    expect(recall.degraded).toBe(false);
-    expect(recall.latencyMs).toBeLessThan(1000);
-  });
-
-  test("retrieval completes under 2000ms for 2000 items", async () => {
-    const conversationId = "conv-bench-2000";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 2000, now);
-
-    const config = makeConfig();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    expect(recall.degraded).toBe(false);
-    expect(recall.latencyMs).toBeLessThan(2000);
-  });
-
-  test("token budget enforcement: maxInjectTokens is respected", async () => {
-    const conversationId = "conv-bench-budget";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 500, now);
-
-    const smallBudget = 200;
-    const config = makeConfig({ maxInjectTokens: smallBudget });
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    // With Qdrant mocked empty, no candidates are found.
-    // Verify the budget cap is still respected.
-    expect(recall.injectedTokens).toBeLessThanOrEqual(smallBudget);
-
-    // Compare against a larger budget
-    const largeBudget = 2000;
-    const largeConfig = makeConfig({ maxInjectTokens: largeBudget });
-    const largeRecall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      largeConfig,
-    );
-
-    expect(largeRecall.injectedTokens).toBeLessThanOrEqual(largeBudget);
-    // With more budget, we should get at least as many tokens
-    expect(largeRecall.injectedTokens).toBeGreaterThanOrEqual(
-      recall.injectedTokens,
-    );
-  });
-
-  test("semantic search is invoked when not early terminated", async () => {
-    const conversationId = "conv-bench-semantic";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 100, now);
-
-    const query = "What do we know about topic-5 and keyword-3?";
-
-    // earlyTermination is always false in the new pipeline, so semantic
-    // search should always be invoked when a query vector is available.
-    semanticSearchCallCount = 0;
-    const config = makeConfig();
-    await buildMemoryRecall(query, conversationId, config);
-
-    expect(semanticSearchCallCount).toBeGreaterThan(0);
-  });
-
-  test("recall.latencyMs tracks wall-clock within 50% tolerance", async () => {
-    const conversationId = "conv-bench-wallclock";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 500, now);
-
-    const config = makeConfig();
-
-    // Use Date.now() to match the timer source used by buildMemoryRecall
-    // (which also uses Date.now()), avoiding precision mismatches between
-    // integer-ms Date.now() and sub-ms performance.now().
-    const wallStart = Date.now();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-    const wallMs = Date.now() - wallStart;
-
-    expect(recall.enabled).toBe(true);
-    // latencyMs may be 0 when the pipeline runs very fast (< 1ms granularity)
-    expect(recall.latencyMs).toBeGreaterThanOrEqual(0);
-
-    // Self-reported latencyMs should agree with wall-clock within 50%.
-    // Tolerance is wide because both sides use Date.now() (integer ms),
-    // so on fast runs the quantization error can be large relative to
-    // total elapsed time.
-    if (wallMs > 0) {
-      const ratio = recall.latencyMs / wallMs;
-      expect(ratio).toBeGreaterThanOrEqual(0.5);
-      expect(ratio).toBeLessThanOrEqual(1.5);
-    }
-  });
-});

package/src/cli/cli-memory.ts

@@ -1,176 +0,0 @@
-import { and, eq } from "drizzle-orm";
-import { v4 as uuid } from "uuid";
-
-import { getDb } from "../memory/db.js";
-import { computeMemoryFingerprint } from "../memory/fingerprint.js";
-import { enqueueMemoryJob } from "../memory/jobs-store.js";
-import { memoryItems } from "../memory/schema.js";
-import { getLogger } from "../util/logger.js";
-import { buildCliProgram } from "./program.js";
-
-const log = getLogger("cli-memory");
-
-/**
- * Build a capability statement for a CLI command.
- * Truncated to 500 chars max (matching the limit used by memory item extraction).
- */
-export function buildCliCapabilityStatement(
-  name: string,
-  description: string,
-): string {
-  let statement = `The "assistant ${name}" CLI command is available. ${description}.`;
-
-  // Truncate to 500 chars max
-  if (statement.length > 500) {
-    statement = statement.slice(0, 500);
-  }
-
-  return statement;
-}
-
-/**
- * Upsert a capability memory item for a CLI command.
- * Best-effort: errors are logged but never thrown.
- */
-export function upsertCliCapabilityMemory(
-  commandName: string,
-  description: string,
-): void {
-  try {
-    const db = getDb();
-    const subject = `cli:${commandName}`;
-    const statement = buildCliCapabilityStatement(commandName, description);
-    const kind = "capability";
-    const scopeId = "default";
-    const confidence = 1.0;
-    const importance = 0.7;
-    const fingerprint = computeMemoryFingerprint(
-      scopeId,
-      kind,
-      subject,
-      statement,
-    );
-    const now = Date.now();
-
-    const existing = db
-      .select()
-      .from(memoryItems)
-      .where(
-        and(
-          eq(memoryItems.kind, kind),
-          eq(memoryItems.subject, subject),
-          eq(memoryItems.scopeId, scopeId),
-        ),
-      )
-      .get();
-
-    if (existing) {
-      if (
-        existing.status === "active" &&
-        existing.fingerprint === fingerprint
-      ) {
-        // Same content — just touch lastSeenAt
-        db.update(memoryItems)
-          .set({ lastSeenAt: now })
-          .where(eq(memoryItems.id, existing.id))
-          .run();
-        return;
-      }
-
-      if (existing.status === "active") {
-        // Content changed — update statement and fingerprint
-        db.update(memoryItems)
-          .set({
-            statement,
-            fingerprint,
-            lastSeenAt: now,
-          })
-          .where(eq(memoryItems.id, existing.id))
-          .run();
-        enqueueMemoryJob("embed_item", { itemId: existing.id });
-        return;
-      }
-
-      // status === "deleted" or other — reactivate
-      db.update(memoryItems)
-        .set({
-          status: "active",
-          statement,
-          fingerprint,
-          lastSeenAt: now,
-          firstSeenAt: now,
-        })
-        .where(eq(memoryItems.id, existing.id))
-        .run();
-      enqueueMemoryJob("embed_item", { itemId: existing.id });
-      return;
-    }
-
-    // No existing — insert new row
-    const id = uuid();
-    db.insert(memoryItems)
-      .values({
-        id,
-        kind,
-        subject,
-        statement,
-        status: "active",
-        confidence,
-        importance,
-        fingerprint,
-        sourceType: "extraction",
-        scopeId,
-        firstSeenAt: now,
-        lastSeenAt: now,
-      })
-      .run();
-    enqueueMemoryJob("embed_item", { itemId: id });
-  } catch (err) {
-    log.warn({ err, commandName }, "Failed to upsert CLI capability memory");
-  }
-}
-
-/**
- * Seed capability memory items for all CLI commands.
- * Prunes stale entries whose commands are no longer registered.
- * Best-effort: errors are logged but never thrown.
- */
-export function seedCliCommandMemories(): void {
-  try {
-    const program = buildCliProgram();
-    const commandNames = new Set<string>();
-
-    for (const cmd of program.commands) {
-      commandNames.add(cmd.name());
-      upsertCliCapabilityMemory(cmd.name(), cmd.description());
-    }
-
-    // Prune stale capability memories for commands no longer registered
-    const db = getDb();
-    const allCapabilities = db
-      .select()
-      .from(memoryItems)
-      .where(
-        and(
-          eq(memoryItems.kind, "capability"),
-          eq(memoryItems.scopeId, "default"),
-          eq(memoryItems.status, "active"),
-        ),
-      )
-      .all();
-
-    const now = Date.now();
-    for (const item of allCapabilities) {
-      if (!item.subject.startsWith("cli:")) continue;
-      const itemCommandName = item.subject.replace("cli:", "");
-      if (!commandNames.has(itemCommandName)) {
-        db.update(memoryItems)
-          .set({ status: "deleted", lastSeenAt: now })
-          .where(eq(memoryItems.id, item.id))
-          .run();
-      }
-    }
-  } catch (err) {
-    log.warn({ err }, "Failed to seed CLI command memories");
-  }
-}

package/src/daemon/conversation-memory.ts

@@ -1,207 +0,0 @@
-import { getConfig } from "../config/loader.js";
-import { estimatePromptTokens } from "../context/token-estimator.js";
-import { buildMemoryQuery } from "../memory/query-builder.js";
-import { computeRecallBudget } from "../memory/retrieval-budget.js";
-import {
-  buildMemoryRecall,
-  injectMemoryRecallAsUserBlock,
-} from "../memory/retriever.js";
-import type { ScopePolicyOverride } from "../memory/search/types.js";
-import type { Message } from "../providers/types.js";
-import type { Provider } from "../providers/types.js";
-import type { ServerMessage } from "./message-protocol.js";
-
-export interface MemoryRecallResult {
-  runMessages: Message[];
-  recall: Awaited<ReturnType<typeof buildMemoryRecall>>;
-}
-
-export interface MemoryPrepareContext {
-  conversationId: string;
-  messages: Message[];
-  systemPrompt: string;
-  provider: Provider;
-  scopeId: string;
-  includeDefaultFallback: boolean;
-  trustClass: "guardian" | "trusted_contact" | "unknown";
-}
-
-/**
- * Returns true when the latest user turn is an internal tool-result-only
- * message (no user-authored text/image content).
- */
-function isToolResultOnlyUserTurn(message: Message | undefined): boolean {
-  return (
-    message?.role === "user" &&
-    message.content.length > 0 &&
-    message.content.every(
-      (block) =>
-        block.type === "tool_result" || block.type === "web_search_tool_result",
-    )
-  );
-}
-
-/**
- * Fast gate that determines whether the current turn warrants memory
- * retrieval. Returns `false` for mechanical no-ops (empty content,
- * tool-result-only) so the full memory pipeline can be skipped.
- * Runs in microseconds — no external calls.
- *
- * Note: We intentionally avoid character-length heuristics here.
- * Short messages like "What did I say?" or "My preferences?" are
- * legitimate memory queries. Per AGENTS.md, judgement calls about
- * message value should be routed through the daemon, not hardcoded.
- */
-export function needsMemory(messages: Message[], content: string): boolean {
-  // Empty or whitespace-only content — mechanical validation, nothing to query
-  if (!content || content.trim().length === 0) return false;
-
-  // Tool-result-only turns (assistant tool loop)
-  const latestMessage = messages[messages.length - 1];
-  if (isToolResultOnlyUserTurn(latestMessage)) return false;
-
-  return true;
-}
-
-/**
- * Build memory recall for a single agent loop turn using the V2 hybrid
- * pipeline. Returns the augmented run messages and metadata for
- * downstream event emission.
- *
- * Memory context is injected as a text content block prepended to the
- * last user message (same pattern as workspace/temporal injections).
- * Stripping is handled by `stripUserTextBlocksByPrefix` matching the
- * `<memory_context __injected>` prefix in `RUNTIME_INJECTION_PREFIXES`.
- */
-export async function prepareMemoryContext(
-  ctx: MemoryPrepareContext,
-  content: string,
-  userMessageId: string,
-  abortSignal: AbortSignal,
-  onEvent: (msg: ServerMessage) => void,
-): Promise<MemoryRecallResult> {
-  // Provenance-based trust gating: untrusted actors skip all memory operations
-  // to prevent untrusted content from influencing memory-augmented responses.
-  const isTrustedActor = ctx.trustClass === "guardian";
-
-  // Build a no-op result that skips the entire memory pipeline.
-  const noopResult = (): MemoryRecallResult => ({
-    runMessages: ctx.messages,
-    recall: {
-      enabled: false,
-      degraded: false,
-      injectedText: "",
-      semanticHits: 0,
-      mergedCount: 0,
-      selectedCount: 0,
-      injectedTokens: 0,
-      latencyMs: 0,
-      topCandidates: [],
-      tier1Count: 0,
-      tier2Count: 0,
-    } as Awaited<ReturnType<typeof buildMemoryRecall>>,
-  });
-
-  if (!isTrustedActor) {
-    return noopResult();
-  }
-
-  // Gate: skip the entire memory pipeline for mechanical no-ops (empty
-  // content, tool-result-only turns).
-  if (!needsMemory(ctx.messages, content)) {
-    return noopResult();
-  }
-
-  const runtimeConfig = getConfig();
-
-  // Memory recall via the V2 hybrid pipeline
-  const recallQuery = buildMemoryQuery(content, ctx.messages);
-  const dynamicBudgetConfig = runtimeConfig.memory?.retrieval?.dynamicBudget;
-  const recallBudget = dynamicBudgetConfig?.enabled
-    ? computeRecallBudget({
-        estimatedPromptTokens: estimatePromptTokens(
-          ctx.messages,
-          ctx.systemPrompt,
-          { providerName: ctx.provider.name },
-        ),
-        maxInputTokens: runtimeConfig.contextWindow.maxInputTokens,
-        targetHeadroomTokens: dynamicBudgetConfig.targetHeadroomTokens,
-        minInjectTokens: dynamicBudgetConfig.minInjectTokens,
-        maxInjectTokens: dynamicBudgetConfig.maxInjectTokens,
-      })
-    : undefined;
-  // Build scope policy override for non-default scopes so retrieval
-  // honours the conversation's memory policy regardless of the global config.
-  const scopePolicyOverride: ScopePolicyOverride | undefined =
-    ctx.scopeId !== "default"
-      ? { scopeId: ctx.scopeId, fallbackToDefault: ctx.includeDefaultFallback }
-      : undefined;
-
-  const recall = await buildMemoryRecall(
-    recallQuery,
-    ctx.conversationId,
-    runtimeConfig,
-    {
-      excludeMessageIds: [userMessageId],
-      signal: abortSignal,
-      maxInjectTokensOverride: recallBudget,
-      scopeId: ctx.scopeId,
-      scopePolicyOverride,
-    },
-  );
-  onEvent({
-    type: "memory_status",
-    enabled: recall.enabled,
-    degraded: recall.degraded,
-    degradation: recall.degradation
-      ? {
-          semanticUnavailable: recall.degradation.semanticUnavailable,
-          reason: recall.degradation.reason,
-          fallbackSources: [...recall.degradation.fallbackSources],
-        }
-      : undefined,
-    reason: recall.reason,
-    provider: recall.provider,
-    model: recall.model,
-  });
-
-  // Inject recall as a text block prepended to the last user message.
-  // When injection text is empty, skip injection entirely.
-  let runMessages = ctx.messages;
-  if (recall.injectedText.length > 0) {
-    const userTail = ctx.messages[ctx.messages.length - 1];
-    if (userTail && userTail.role === "user") {
-      runMessages = injectMemoryRecallAsUserBlock(
-        ctx.messages,
-        recall.injectedText,
-      );
-      onEvent({
-        type: "memory_recalled",
-        provider: recall.provider ?? "unknown",
-        model: recall.model ?? "unknown",
-        degradation: recall.degradation
-          ? {
-              semanticUnavailable: recall.degradation.semanticUnavailable,
-              reason: recall.degradation.reason,
-              fallbackSources: [...recall.degradation.fallbackSources],
-            }
-          : undefined,
-        semanticHits: recall.semanticHits,
-        tier1Count: recall.tier1Count ?? 0,
-        tier2Count: recall.tier2Count ?? 0,
-        hybridSearchLatencyMs: recall.hybridSearchMs ?? 0,
-        sparseVectorUsed: recall.sparseVectorUsed ?? false,
-        mergedCount: recall.mergedCount,
-        selectedCount: recall.selectedCount,
-        injectedTokens: recall.injectedTokens,
-        latencyMs: recall.latencyMs,
-        topCandidates: recall.topCandidates,
-      });
-    }
-  }
-
-  return {
-    runMessages,
-    recall,
-  };
-}