@vellumai/assistant 0.5.16 → 0.6.0

package/src/__tests__/memory-retrieval.benchmark.test.ts

@@ -1,295 +0,0 @@
-/**
- * Memory Retrieval Benchmark
- *
- * Measures end-to-end memory recall time with varying database sizes.
- * Validates latency stays within acceptable bounds and token budget
- * enforcement works correctly.
- *
- * The pipeline uses hybrid search (Qdrant) only. With Qdrant mocked empty,
- * no candidates are found and injectedText is empty. The tests verify
- * pipeline completion, latency bounds, and token budget enforcement.
- */
-import {
-  afterAll,
-  beforeAll,
-  beforeEach,
-  describe,
-  expect,
-  mock,
-  test,
-} from "bun:test";
-
-mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
-}));
-
-// Counter for semantic search invocations
-let semanticSearchCallCount = 0;
-
-mock.module("../memory/search/semantic.js", () => ({
-  semanticSearch: async () => {
-    semanticSearchCallCount++;
-    return [];
-  },
-  isQdrantConnectionError: () => false,
-}));
-
-mock.module("../memory/embedding-backend.js", () => ({
-  getMemoryBackendStatus: async (config: { memory: { enabled: boolean } }) => ({
-    enabled: config.memory.enabled,
-    degraded: false,
-    provider: "local",
-    model: "mock-embedding",
-    reason: null,
-  }),
-  embedWithBackend: async () => ({
-    provider: "local" as const,
-    model: "mock-embedding",
-    vectors: [new Array(1536).fill(0)],
-  }),
-  generateSparseEmbedding: () => ({ indices: [], values: [] }),
-  logMemoryEmbeddingWarning: () => {},
-}));
-
-import { DEFAULT_CONFIG } from "../config/defaults.js";
-import type { AssistantConfig } from "../config/types.js";
-import { getDb, initializeDb, resetDb } from "../memory/db.js";
-import { buildMemoryRecall } from "../memory/retriever.js";
-import { conversations, memorySegments, messages } from "../memory/schema.js";
-
-function seedMemoryItems(
-  conversationId: string,
-  count: number,
-  now: number,
-): void {
-  const db = getDb();
-  db.insert(conversations)
-    .values({
-      id: conversationId,
-      title: null,
-      createdAt: now,
-      updatedAt: now,
-      totalInputTokens: 0,
-      totalOutputTokens: 0,
-      totalEstimatedCost: 0,
-      contextSummary: null,
-      contextCompactedMessageCount: 0,
-      contextCompactedAt: null,
-    })
-    .run();
-
-  for (let i = 0; i < count; i++) {
-    const msgId = `msg-${conversationId}-${i}`;
-    const text = `Memory item ${i}: information about topic-${
-      i % 20
-    } including keyword-${i % 10} details.`;
-    db.insert(messages)
-      .values({
-        id: msgId,
-        conversationId,
-        role: i % 2 === 0 ? "user" : "assistant",
-        content: JSON.stringify([{ type: "text", text }]),
-        createdAt: now + i,
-      })
-      .run();
-    db.insert(memorySegments)
-      .values({
-        id: `seg-${conversationId}-${i}`,
-        messageId: msgId,
-        conversationId,
-        role: i % 2 === 0 ? "user" : "assistant",
-        segmentIndex: 0,
-        text,
-        tokenEstimate: 20,
-        scopeId: "default",
-        createdAt: now + i,
-        updatedAt: now + i,
-      })
-      .run();
-  }
-}
-
-function makeConfig(overrides?: { maxInjectTokens?: number }): AssistantConfig {
-  return {
-    ...DEFAULT_CONFIG,
-    memory: {
-      ...DEFAULT_CONFIG.memory,
-      embeddings: {
-        ...DEFAULT_CONFIG.memory.embeddings,
-        provider: "local" as const,
-        required: false,
-      },
-      retrieval: {
-        ...DEFAULT_CONFIG.memory.retrieval,
-        maxInjectTokens: overrides?.maxInjectTokens ?? 750,
-        dynamicBudget: {
-          enabled: false,
-          minInjectTokens: 160,
-          maxInjectTokens: overrides?.maxInjectTokens ?? 750,
-          targetHeadroomTokens: 900,
-        },
-      },
-    },
-  };
-}
-
-describe("Memory retrieval benchmark", () => {
-  beforeAll(() => {
-    initializeDb();
-  });
-
-  beforeEach(() => {
-    const db = getDb();
-    db.run("DELETE FROM memory_item_sources");
-    db.run("DELETE FROM memory_embeddings");
-    db.run("DELETE FROM memory_items");
-
-    db.run("DELETE FROM memory_segments");
-    db.run("DELETE FROM messages");
-    db.run("DELETE FROM conversations");
-    db.run("DELETE FROM memory_jobs");
-    db.run("DELETE FROM memory_checkpoints");
-  });
-
-  afterAll(() => {
-    resetDb();
-  });
-
-  test("retrieval completes under 500ms for 100 items", async () => {
-    const conversationId = "conv-bench-100";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 100, now);
-
-    const config = makeConfig();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    expect(recall.degraded).toBe(false);
-    // Relaxed threshold — guards against severe regressions, not precise benchmarking
-    expect(recall.latencyMs).toBeLessThan(500);
-  });
-
-  test("retrieval completes under 1000ms for 500 items", async () => {
-    const conversationId = "conv-bench-500";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 500, now);
-
-    const config = makeConfig();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    expect(recall.degraded).toBe(false);
-    expect(recall.latencyMs).toBeLessThan(1000);
-  });
-
-  test("retrieval completes under 2000ms for 2000 items", async () => {
-    const conversationId = "conv-bench-2000";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 2000, now);
-
-    const config = makeConfig();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    expect(recall.degraded).toBe(false);
-    expect(recall.latencyMs).toBeLessThan(2000);
-  });
-
-  test("token budget enforcement: maxInjectTokens is respected", async () => {
-    const conversationId = "conv-bench-budget";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 500, now);
-
-    const smallBudget = 200;
-    const config = makeConfig({ maxInjectTokens: smallBudget });
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-
-    expect(recall.enabled).toBe(true);
-    // With Qdrant mocked empty, no candidates are found.
-    // Verify the budget cap is still respected.
-    expect(recall.injectedTokens).toBeLessThanOrEqual(smallBudget);
-
-    // Compare against a larger budget
-    const largeBudget = 2000;
-    const largeConfig = makeConfig({ maxInjectTokens: largeBudget });
-    const largeRecall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      largeConfig,
-    );
-
-    expect(largeRecall.injectedTokens).toBeLessThanOrEqual(largeBudget);
-    // With more budget, we should get at least as many tokens
-    expect(largeRecall.injectedTokens).toBeGreaterThanOrEqual(
-      recall.injectedTokens,
-    );
-  });
-
-  test("semantic search is invoked when not early terminated", async () => {
-    const conversationId = "conv-bench-semantic";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 100, now);
-
-    const query = "What do we know about topic-5 and keyword-3?";
-
-    // earlyTermination is always false in the new pipeline, so semantic
-    // search should always be invoked when a query vector is available.
-    semanticSearchCallCount = 0;
-    const config = makeConfig();
-    await buildMemoryRecall(query, conversationId, config);
-
-    expect(semanticSearchCallCount).toBeGreaterThan(0);
-  });
-
-  test("recall.latencyMs tracks wall-clock within 50% tolerance", async () => {
-    const conversationId = "conv-bench-wallclock";
-    const now = 1_700_500_000_000;
-    seedMemoryItems(conversationId, 500, now);
-
-    const config = makeConfig();
-
-    // Use Date.now() to match the timer source used by buildMemoryRecall
-    // (which also uses Date.now()), avoiding precision mismatches between
-    // integer-ms Date.now() and sub-ms performance.now().
-    const wallStart = Date.now();
-    const recall = await buildMemoryRecall(
-      "What do we know about topic-5 and keyword-3?",
-      conversationId,
-      config,
-    );
-    const wallMs = Date.now() - wallStart;
-
-    expect(recall.enabled).toBe(true);
-    // latencyMs may be 0 when the pipeline runs very fast (< 1ms granularity)
-    expect(recall.latencyMs).toBeGreaterThanOrEqual(0);
-
-    // Self-reported latencyMs should agree with wall-clock within 50%.
-    // Tolerance is wide because both sides use Date.now() (integer ms),
-    // so on fast runs the quantization error can be large relative to
-    // total elapsed time.
-    if (wallMs > 0) {
-      const ratio = recall.latencyMs / wallMs;
-      expect(ratio).toBeGreaterThanOrEqual(0.5);
-      expect(ratio).toBeLessThanOrEqual(1.5);
-    }
-  });
-});

package/src/daemon/conversation-memory.ts

@@ -1,207 +0,0 @@
-import { getConfig } from "../config/loader.js";
-import { estimatePromptTokens } from "../context/token-estimator.js";
-import { buildMemoryQuery } from "../memory/query-builder.js";
-import { computeRecallBudget } from "../memory/retrieval-budget.js";
-import {
-  buildMemoryRecall,
-  injectMemoryRecallAsUserBlock,
-} from "../memory/retriever.js";
-import type { ScopePolicyOverride } from "../memory/search/types.js";
-import type { Message } from "../providers/types.js";
-import type { Provider } from "../providers/types.js";
-import type { ServerMessage } from "./message-protocol.js";
-
-export interface MemoryRecallResult {
-  runMessages: Message[];
-  recall: Awaited<ReturnType<typeof buildMemoryRecall>>;
-}
-
-export interface MemoryPrepareContext {
-  conversationId: string;
-  messages: Message[];
-  systemPrompt: string;
-  provider: Provider;
-  scopeId: string;
-  includeDefaultFallback: boolean;
-  trustClass: "guardian" | "trusted_contact" | "unknown";
-}
-
-/**
- * Returns true when the latest user turn is an internal tool-result-only
- * message (no user-authored text/image content).
- */
-function isToolResultOnlyUserTurn(message: Message | undefined): boolean {
-  return (
-    message?.role === "user" &&
-    message.content.length > 0 &&
-    message.content.every(
-      (block) =>
-        block.type === "tool_result" || block.type === "web_search_tool_result",
-    )
-  );
-}
-
-/**
- * Fast gate that determines whether the current turn warrants memory
- * retrieval. Returns `false` for mechanical no-ops (empty content,
- * tool-result-only) so the full memory pipeline can be skipped.
- * Runs in microseconds — no external calls.
- *
- * Note: We intentionally avoid character-length heuristics here.
- * Short messages like "What did I say?" or "My preferences?" are
- * legitimate memory queries. Per AGENTS.md, judgement calls about
- * message value should be routed through the daemon, not hardcoded.
- */
-export function needsMemory(messages: Message[], content: string): boolean {
-  // Empty or whitespace-only content — mechanical validation, nothing to query
-  if (!content || content.trim().length === 0) return false;
-
-  // Tool-result-only turns (assistant tool loop)
-  const latestMessage = messages[messages.length - 1];
-  if (isToolResultOnlyUserTurn(latestMessage)) return false;
-
-  return true;
-}
-
-/**
- * Build memory recall for a single agent loop turn using the V2 hybrid
- * pipeline. Returns the augmented run messages and metadata for
- * downstream event emission.
- *
- * Memory context is injected as a text content block prepended to the
- * last user message (same pattern as workspace/temporal injections).
- * Stripping is handled by `stripUserTextBlocksByPrefix` matching the
- * `<memory_context __injected>` prefix in `RUNTIME_INJECTION_PREFIXES`.
- */
-export async function prepareMemoryContext(
-  ctx: MemoryPrepareContext,
-  content: string,
-  userMessageId: string,
-  abortSignal: AbortSignal,
-  onEvent: (msg: ServerMessage) => void,
-): Promise<MemoryRecallResult> {
-  // Provenance-based trust gating: untrusted actors skip all memory operations
-  // to prevent untrusted content from influencing memory-augmented responses.
-  const isTrustedActor = ctx.trustClass === "guardian";
-
-  // Build a no-op result that skips the entire memory pipeline.
-  const noopResult = (): MemoryRecallResult => ({
-    runMessages: ctx.messages,
-    recall: {
-      enabled: false,
-      degraded: false,
-      injectedText: "",
-      semanticHits: 0,
-      mergedCount: 0,
-      selectedCount: 0,
-      injectedTokens: 0,
-      latencyMs: 0,
-      topCandidates: [],
-      tier1Count: 0,
-      tier2Count: 0,
-    } as Awaited<ReturnType<typeof buildMemoryRecall>>,
-  });
-
-  if (!isTrustedActor) {
-    return noopResult();
-  }
-
-  // Gate: skip the entire memory pipeline for mechanical no-ops (empty
-  // content, tool-result-only turns).
-  if (!needsMemory(ctx.messages, content)) {
-    return noopResult();
-  }
-
-  const runtimeConfig = getConfig();
-
-  // Memory recall via the V2 hybrid pipeline
-  const recallQuery = buildMemoryQuery(content, ctx.messages);
-  const dynamicBudgetConfig = runtimeConfig.memory?.retrieval?.dynamicBudget;
-  const recallBudget = dynamicBudgetConfig?.enabled
-    ? computeRecallBudget({
-        estimatedPromptTokens: estimatePromptTokens(
-          ctx.messages,
-          ctx.systemPrompt,
-          { providerName: ctx.provider.name },
-        ),
-        maxInputTokens: runtimeConfig.contextWindow.maxInputTokens,
-        targetHeadroomTokens: dynamicBudgetConfig.targetHeadroomTokens,
-        minInjectTokens: dynamicBudgetConfig.minInjectTokens,
-        maxInjectTokens: dynamicBudgetConfig.maxInjectTokens,
-      })
-    : undefined;
-  // Build scope policy override for non-default scopes so retrieval
-  // honours the conversation's memory policy regardless of the global config.
-  const scopePolicyOverride: ScopePolicyOverride | undefined =
-    ctx.scopeId !== "default"
-      ? { scopeId: ctx.scopeId, fallbackToDefault: ctx.includeDefaultFallback }
-      : undefined;
-
-  const recall = await buildMemoryRecall(
-    recallQuery,
-    ctx.conversationId,
-    runtimeConfig,
-    {
-      excludeMessageIds: [userMessageId],
-      signal: abortSignal,
-      maxInjectTokensOverride: recallBudget,
-      scopeId: ctx.scopeId,
-      scopePolicyOverride,
-    },
-  );
-  onEvent({
-    type: "memory_status",
-    enabled: recall.enabled,
-    degraded: recall.degraded,
-    degradation: recall.degradation
-      ? {
-          semanticUnavailable: recall.degradation.semanticUnavailable,
-          reason: recall.degradation.reason,
-          fallbackSources: [...recall.degradation.fallbackSources],
-        }
-      : undefined,
-    reason: recall.reason,
-    provider: recall.provider,
-    model: recall.model,
-  });
-
-  // Inject recall as a text block prepended to the last user message.
-  // When injection text is empty, skip injection entirely.
-  let runMessages = ctx.messages;
-  if (recall.injectedText.length > 0) {
-    const userTail = ctx.messages[ctx.messages.length - 1];
-    if (userTail && userTail.role === "user") {
-      runMessages = injectMemoryRecallAsUserBlock(
-        ctx.messages,
-        recall.injectedText,
-      );
-      onEvent({
-        type: "memory_recalled",
-        provider: recall.provider ?? "unknown",
-        model: recall.model ?? "unknown",
-        degradation: recall.degradation
-          ? {
-              semanticUnavailable: recall.degradation.semanticUnavailable,
-              reason: recall.degradation.reason,
-              fallbackSources: [...recall.degradation.fallbackSources],
-            }
-          : undefined,
-        semanticHits: recall.semanticHits,
-        tier1Count: recall.tier1Count ?? 0,
-        tier2Count: recall.tier2Count ?? 0,
-        hybridSearchLatencyMs: recall.hybridSearchMs ?? 0,
-        sparseVectorUsed: recall.sparseVectorUsed ?? false,
-        mergedCount: recall.mergedCount,
-        selectedCount: recall.selectedCount,
-        injectedTokens: recall.injectedTokens,
-        latencyMs: recall.latencyMs,
-        topCandidates: recall.topCandidates,
-      });
-    }
-  }
-
-  return {
-    runMessages,
-    recall,
-  };
-}

package/src/memory/conversation-starters-cadence.ts

@@ -1,74 +0,0 @@
-/**
- * Cadence logic for conversation starters generation.
- *
- * Decides whether a new generation job should be enqueued based on how many
- * active memory items have accumulated since the last generation.
- */
-
-import { and, eq, inArray, like } from "drizzle-orm";
-
-import { getLogger } from "../util/logger.js";
-import { getDb } from "./db.js";
-import { enqueueMemoryJob } from "./jobs-store.js";
-import { rawGet } from "./raw-query.js";
-import { memoryCheckpoints, memoryJobs } from "./schema.js";
-
-const log = getLogger("conversation-starters-cadence");
-
-/**
- * Check whether enough new memory items have accumulated to justify
- * generating a fresh batch of conversation starters.
- */
-export function maybeEnqueueConversationStartersJob(scopeId: string): void {
-  const db = getDb();
-
-  // Count total active memory items
-  const countRow = rawGet<{ c: number }>(
-    `SELECT COUNT(*) AS c FROM memory_items WHERE status = 'active' AND scope_id = ?`,
-    scopeId,
-  );
-  const totalActive = countRow?.c ?? 0;
-  if (totalActive === 0) return;
-
-  // Read checkpoint: item count at last generation (scoped so each scope tracks independently)
-  const checkpointKey = `conversation_starters:item_count_at_last_gen:${scopeId}`;
-  const checkpoint = db
-    .select({ value: memoryCheckpoints.value })
-    .from(memoryCheckpoints)
-    .where(eq(memoryCheckpoints.key, checkpointKey))
-    .get();
-  const lastCount = checkpoint ? parseInt(checkpoint.value, 10) : 0;
-
-  // Cadence formula
-  let threshold: number;
-  if (totalActive <= 10) {
-    threshold = 1;
-  } else if (totalActive <= 50) {
-    threshold = 5;
-  } else {
-    threshold = 10;
-  }
-
-  const delta = totalActive - lastCount;
-  if (delta < threshold) return;
-
-  // Dedup: don't enqueue if a pending/running job for this scope already exists
-  const existing = db
-    .select({ id: memoryJobs.id })
-    .from(memoryJobs)
-    .where(
-      and(
-        eq(memoryJobs.type, "generate_conversation_starters"),
-        inArray(memoryJobs.status, ["pending", "running"]),
-        like(memoryJobs.payload, `%"scopeId":"${scopeId}"%`),
-      ),
-    )
-    .get();
-  if (existing) return;
-
-  enqueueMemoryJob("generate_conversation_starters", { scopeId });
-  log.info(
-    { totalActive, lastCount, delta, threshold, scopeId },
-    "Enqueued conversation starters generation job",
-  );
-}