@vellumai/assistant 0.4.49 → 0.4.51

package/src/memory/fts-reconciler.ts

@@ -1,165 +0,0 @@
-import { getLogger } from "../util/logger.js";
-import { rawGet, rawRun } from "./db.js";
-
-const log = getLogger("fts-reconciler");
-
-export interface FtsReconciliationResult {
-  table: string;
-  baseCount: number;
-  ftsCount: number;
-  missingInserted: number;
-  orphansRemoved: number;
-  staleRefreshed: number;
-}
-
-/**
- * Reconcile a single FTS index against its base table. Detects missing entries
- * (rows in the base table with no corresponding FTS row) and orphaned entries
- * (FTS rows whose base table row no longer exists), then repairs both.
- *
- * This is lighter than a full rebuild — it only touches the delta rather than
- * wiping and re-inserting the entire index.
- */
-function reconcileTable(opts: {
-  ftsTable: string;
-  ftsIdColumn: string;
-  ftsContentColumn: string;
-  baseTable: string;
-  baseIdColumn: string;
-  baseContentColumn: string;
-}): FtsReconciliationResult {
-  const {
-    ftsTable,
-    ftsIdColumn,
-    ftsContentColumn,
-    baseTable,
-    baseIdColumn,
-    baseContentColumn,
-  } = opts;
-
-  const baseCount = (
-    rawGet<{ c: number }>(`SELECT COUNT(*) AS c FROM ${baseTable}`) ?? { c: 0 }
-  ).c;
-  const ftsCount = (
-    rawGet<{ c: number }>(`SELECT COUNT(*) AS c FROM ${ftsTable}`) ?? { c: 0 }
-  ).c;
-
-  // Find base table rows missing from the FTS index
-  const missingInserted = rawRun(/*sql*/ `
-    INSERT INTO ${ftsTable}(${ftsIdColumn}, ${ftsContentColumn})
-    SELECT b.${baseIdColumn}, b.${baseContentColumn}
-    FROM ${baseTable} b
-    LEFT JOIN ${ftsTable} f ON f.${ftsIdColumn} = b.${baseIdColumn}
-    WHERE f.${ftsIdColumn} IS NULL
-  `);
-
-  // Find FTS rows whose base table row no longer exists
-  const orphansRemoved = rawRun(/*sql*/ `
-    DELETE FROM ${ftsTable}
-    WHERE ${ftsIdColumn} IN (
-      SELECT f.${ftsIdColumn}
-      FROM ${ftsTable} f
-      LEFT JOIN ${baseTable} b ON b.${baseIdColumn} = f.${ftsIdColumn}
-      WHERE b.${baseIdColumn} IS NULL
-    )
-  `);
-
-  // Refresh FTS rows whose content is stale (base row was updated but the
-  // update trigger didn't fire or was missing). Delete-then-insert is the
-  // standard FTS5 update pattern.
-  const staleDeleted = rawRun(/*sql*/ `
-    DELETE FROM ${ftsTable}
-    WHERE ${ftsIdColumn} IN (
-      SELECT f.${ftsIdColumn}
-      FROM ${ftsTable} f
-      JOIN ${baseTable} b ON b.${baseIdColumn} = f.${ftsIdColumn}
-      WHERE b.${baseContentColumn} IS NOT f.${ftsContentColumn}
-    )
-  `);
-  if (staleDeleted > 0) {
-    rawRun(/*sql*/ `
-      INSERT INTO ${ftsTable}(${ftsIdColumn}, ${ftsContentColumn})
-      SELECT b.${baseIdColumn}, b.${baseContentColumn}
-      FROM ${baseTable} b
-      LEFT JOIN ${ftsTable} f ON f.${ftsIdColumn} = b.${baseIdColumn}
-      WHERE f.${ftsIdColumn} IS NULL
-    `);
-  }
-
-  return {
-    table: ftsTable,
-    baseCount,
-    ftsCount,
-    missingInserted,
-    orphansRemoved,
-    staleRefreshed: staleDeleted,
-  };
-}
-
-/**
- * Reconcile all FTS indexes. Returns results for each table so callers can
- * inspect what was repaired.
- */
-export function reconcileFtsIndexes(): FtsReconciliationResult[] {
-  const results: FtsReconciliationResult[] = [];
-  const errors: unknown[] = [];
-
-  // memory_segment_fts tracks memory_segments
-  try {
-    const memResult = reconcileTable({
-      ftsTable: "memory_segment_fts",
-      ftsIdColumn: "segment_id",
-      ftsContentColumn: "text",
-      baseTable: "memory_segments",
-      baseIdColumn: "id",
-      baseContentColumn: "text",
-    });
-    results.push(memResult);
-    if (
-      memResult.missingInserted > 0 ||
-      memResult.orphansRemoved > 0 ||
-      memResult.staleRefreshed > 0
-    ) {
-      log.info(memResult, "Reconciled memory_segment_fts");
-    } else {
-      log.debug(memResult, "memory_segment_fts is in sync");
-    }
-  } catch (err) {
-    log.error({ err }, "Failed to reconcile memory_segment_fts");
-    errors.push(err);
-  }
-
-  // messages_fts tracks messages
-  try {
-    const msgResult = reconcileTable({
-      ftsTable: "messages_fts",
-      ftsIdColumn: "message_id",
-      ftsContentColumn: "content",
-      baseTable: "messages",
-      baseIdColumn: "id",
-      baseContentColumn: "content",
-    });
-    results.push(msgResult);
-    if (
-      msgResult.missingInserted > 0 ||
-      msgResult.orphansRemoved > 0 ||
-      msgResult.staleRefreshed > 0
-    ) {
-      log.info(msgResult, "Reconciled messages_fts");
-    } else {
-      log.debug(msgResult, "messages_fts is in sync");
-    }
-  } catch (err) {
-    log.error({ err }, "Failed to reconcile messages_fts");
-    errors.push(err);
-  }
-
-  if (errors.length > 0) {
-    throw new AggregateError(
-      errors,
-      `FTS reconciliation failed for ${errors.length} table(s)`,
-    );
-  }
-
-  return results;
-}

package/src/memory/job-handlers/conflict.ts

@@ -1,200 +0,0 @@
-import { and, asc, eq, inArray, lt, ne } from "drizzle-orm";
-
-import type { AssistantConfig } from "../../config/types.js";
-import { getLogger } from "../../util/logger.js";
-import { resolveConflictClarification } from "../clarification-resolver.js";
-import {
-  areStatementsCoherent,
-  computeConflictRelevance,
-  looksLikeClarificationReply,
-  shouldAttemptConflictResolution,
-} from "../conflict-intent.js";
-import {
-  isConflictKindPairEligible,
-  isConflictUserEvidenced,
-  isStatementConflictEligible,
-} from "../conflict-policy.js";
-import {
-  applyConflictResolution,
-  listPendingConflictDetails,
-  resolveConflict,
-} from "../conflict-store.js";
-import { getDb } from "../db.js";
-import { asPositiveMs, asString } from "../job-utils.js";
-import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
-import { extractTextFromStoredMessageContent } from "../message-content.js";
-import { memoryItemConflicts, messages } from "../schema.js";
-
-const log = getLogger("memory-jobs-worker");
-
-const CLEANUP_BATCH_LIMIT = 250;
-
-export async function resolvePendingConflictsForMessageJob(
-  job: MemoryJob,
-  config: AssistantConfig,
-): Promise<void> {
-  if (!config.memory.conflicts.enabled) return;
-  const messageId = asString(job.payload.messageId);
-  if (!messageId) return;
-  const scopeId = asString(job.payload.scopeId) ?? "default";
-  const db = getDb();
-  const message = db
-    .select({
-      id: messages.id,
-      role: messages.role,
-      content: messages.content,
-      createdAt: messages.createdAt,
-    })
-    .from(messages)
-    .where(eq(messages.id, messageId))
-    .get();
-  if (!message || message.role !== "user") return;
-
-  const userMessage = extractTextFromStoredMessageContent(
-    message.content,
-  ).trim();
-  if (userMessage.length === 0) return;
-  const clarificationReply = looksLikeClarificationReply(userMessage);
-  if (!clarificationReply) return;
-
-  const pending = listPendingConflictDetails(scopeId, 25);
-
-  // Dismiss non-actionable conflicts (kind or statement policy)
-  const conflictableKinds = config.memory.conflicts.conflictableKinds;
-  for (const conflict of pending) {
-    const kindEligible = isConflictKindPairEligible(
-      conflict.existingKind,
-      conflict.candidateKind,
-      { conflictableKinds },
-    );
-    if (
-      !kindEligible ||
-      !isStatementConflictEligible(
-        conflict.existingKind,
-        conflict.existingStatement,
-        { conflictableKinds },
-      ) ||
-      !isStatementConflictEligible(
-        conflict.candidateKind,
-        conflict.candidateStatement,
-        { conflictableKinds },
-      )
-    ) {
-      resolveConflict(conflict.id, {
-        status: "dismissed",
-        resolutionNote: "Dismissed by conflict policy (transient/non-durable).",
-      });
-    } else if (
-      !isConflictUserEvidenced(
-        conflict.existingVerificationState,
-        conflict.candidateVerificationState,
-      )
-    ) {
-      resolveConflict(conflict.id, {
-        status: "dismissed",
-        resolutionNote:
-          "Dismissed by conflict policy (no user-evidenced provenance).",
-      });
-    } else if (
-      !areStatementsCoherent(
-        conflict.existingStatement,
-        conflict.candidateStatement,
-      )
-    ) {
-      resolveConflict(conflict.id, {
-        status: "dismissed",
-        resolutionNote:
-          "Dismissed by conflict policy (incoherent — zero statement overlap).",
-      });
-    }
-  }
-
-  // Re-fetch after dismissal
-  const actionablePending = listPendingConflictDetails(scopeId, 25);
-  const eligible = actionablePending.filter(
-    (conflict) => conflict.createdAt <= message.createdAt,
-  );
-  if (eligible.length === 0) return;
-  const candidates = eligible.filter((conflict) => {
-    const relevance = computeConflictRelevance(userMessage, conflict);
-    return shouldAttemptConflictResolution({
-      clarificationReply,
-      relevance,
-    });
-  });
-  if (candidates.length === 0) return;
-
-  let resolvedCount = 0;
-  for (const conflict of candidates) {
-    const resolution = await resolveConflictClarification(
-      {
-        existingStatement: conflict.existingStatement,
-        candidateStatement: conflict.candidateStatement,
-        userMessage,
-      },
-      { timeoutMs: config.memory.conflicts.resolverLlmTimeoutMs },
-    );
-    if (resolution.resolution === "still_unclear") continue;
-    const resolved = applyConflictResolution({
-      conflictId: conflict.id,
-      resolution: resolution.resolution,
-      mergedStatement:
-        resolution.resolution === "merge" ? resolution.resolvedStatement : null,
-      resolutionNote: `Background message resolver (${resolution.strategy}): ${resolution.explanation}`,
-    });
-    if (resolved) resolvedCount += 1;
-  }
-
-  log.debug(
-    {
-      messageId,
-      scopeId,
-      pendingConflicts: pending.length,
-      eligibleConflicts: eligible.length,
-      candidateConflicts: candidates.length,
-      resolvedConflicts: resolvedCount,
-    },
-    "Processed pending conflict resolution job",
-  );
-}
-
-export function cleanupResolvedConflictsJob(
-  job: MemoryJob,
-  config: AssistantConfig,
-): void {
-  const db = getDb();
-  const retentionMs =
-    asPositiveMs(job.payload.retentionMs) ??
-    config.memory.cleanup.resolvedConflictRetentionMs;
-  const cutoff = Date.now() - retentionMs;
-  const stale = db
-    .select({ id: memoryItemConflicts.id })
-    .from(memoryItemConflicts)
-    .where(
-      and(
-        ne(memoryItemConflicts.status, "pending_clarification"),
-        lt(memoryItemConflicts.resolvedAt, cutoff),
-      ),
-    )
-    .orderBy(asc(memoryItemConflicts.resolvedAt), asc(memoryItemConflicts.id))
-    .limit(CLEANUP_BATCH_LIMIT)
-    .all();
-  if (stale.length === 0) return;
-
-  const ids = stale.map((row) => row.id);
-  db.delete(memoryItemConflicts)
-    .where(inArray(memoryItemConflicts.id, ids))
-    .run();
-  if (stale.length === CLEANUP_BATCH_LIMIT) {
-    enqueueMemoryJob("cleanup_resolved_conflicts", { retentionMs });
-  }
-
-  log.debug(
-    {
-      removedConflicts: stale.length,
-      retentionMs,
-      cutoff,
-    },
-    "Cleaned up resolved memory conflicts",
-  );
-}

package/src/memory/profile-compiler.ts

@@ -1,195 +0,0 @@
-import { and, desc, eq, inArray, isNull } from "drizzle-orm";
-
-import { getConfig } from "../config/loader.js";
-import { estimateTextTokens } from "../context/token-estimator.js";
-import { getDb } from "./db.js";
-import { memoryItems } from "./schema.js";
-
-const PROFILE_KIND_ALLOWLIST = [
-  "profile",
-  "preference",
-  "constraint",
-  "instruction",
-  "style",
-] as const;
-
-const TRUST_RANK: Record<string, number> = {
-  user_confirmed: 3,
-  user_reported: 2,
-  assistant_inferred: 1,
-};
-
-export interface CompileProfileOptions {
-  scopeId?: string;
-  maxInjectTokensOverride?: number;
-  /** When true and scopeId is not 'default', query both the given scope and 'default'. */
-  includeDefaultFallback?: boolean;
-}
-
-export interface CompiledProfile {
-  text: string;
-  sourceCount: number;
-  selectedCount: number;
-  budgetTokens: number;
-  tokenEstimate: number;
-}
-
-interface ProfileCandidate {
-  kind: string;
-  subject: string;
-  statement: string;
-  verificationState: string;
-  confidence: number;
-  importance: number | null;
-  lastSeenAt: number;
-  firstSeenAt: number;
-  scopeId: string;
-}
-
-export function compileDynamicProfile(
-  options?: CompileProfileOptions,
-): CompiledProfile {
-  const config = getConfig();
-  const profileConfig = config.memory.profile;
-  const scopeId = options?.scopeId ?? "default";
-  const budgetTokens = Math.max(
-    0,
-    Math.floor(
-      options?.maxInjectTokensOverride ?? profileConfig.maxInjectTokens,
-    ),
-  );
-  if (!profileConfig.enabled || budgetTokens <= 0) {
-    return {
-      text: "",
-      sourceCount: 0,
-      selectedCount: 0,
-      budgetTokens,
-      tokenEstimate: 0,
-    };
-  }
-
-  const db = getDb();
-  const shouldFallback =
-    options?.includeDefaultFallback === true && scopeId !== "default";
-  const scopeFilter = shouldFallback
-    ? inArray(memoryItems.scopeId, [scopeId, "default"])
-    : eq(memoryItems.scopeId, scopeId);
-  const rows = db
-    .select({
-      kind: memoryItems.kind,
-      subject: memoryItems.subject,
-      statement: memoryItems.statement,
-      verificationState: memoryItems.verificationState,
-      confidence: memoryItems.confidence,
-      importance: memoryItems.importance,
-      lastSeenAt: memoryItems.lastSeenAt,
-      firstSeenAt: memoryItems.firstSeenAt,
-      scopeId: memoryItems.scopeId,
-    })
-    .from(memoryItems)
-    .where(
-      and(
-        scopeFilter,
-        eq(memoryItems.status, "active"),
-        isNull(memoryItems.invalidAt),
-        inArray(memoryItems.kind, [...PROFILE_KIND_ALLOWLIST]),
-      ),
-    )
-    .orderBy(desc(memoryItems.lastSeenAt))
-    .all();
-
-  const nowMs = Date.now();
-  const trusted = rows
-    .filter((row) => TRUST_RANK[row.verificationState] !== undefined)
-    .sort((a, b) =>
-      compareProfileCandidates(
-        a,
-        b,
-        nowMs,
-        shouldFallback ? scopeId : undefined,
-      ),
-    );
-
-  const selectedLines: string[] = [];
-  const seenKeys = new Set<string>();
-  for (const candidate of trusted) {
-    const subject = normalizeWhitespace(candidate.subject, 80);
-    const statement = normalizeWhitespace(candidate.statement, 220);
-    if (!subject || !statement) continue;
-    const dedupeKey = `${candidate.kind}|${subject.toLowerCase()}`;
-    if (seenKeys.has(dedupeKey)) continue;
-
-    const line = `- ${subject}: ${statement}`;
-    const tentative = renderProfileText([...selectedLines, line]);
-    if (estimateTextTokens(tentative) > budgetTokens) continue;
-    seenKeys.add(dedupeKey);
-    selectedLines.push(line);
-  }
-
-  const text = renderProfileText(selectedLines);
-  const tokenEstimate = text ? estimateTextTokens(text) : 0;
-  return {
-    text,
-    sourceCount: trusted.length,
-    selectedCount: selectedLines.length,
-    budgetTokens,
-    tokenEstimate,
-  };
-}
-
-/** Half-life for recency decay — items seen this many ms ago score ~0.5. (30 days) */
-const RECENCY_HALF_LIFE_MS = 30 * 24 * 60 * 60 * 1000;
-
-function recencyScore(lastSeenAt: number, nowMs: number): number {
-  const ageMs = Math.max(0, nowMs - lastSeenAt);
-  return Math.pow(0.5, ageMs / RECENCY_HALF_LIFE_MS);
-}
-
-function candidateRankScore(c: ProfileCandidate, nowMs: number): number {
-  const importance = c.importance ?? 0.5;
-  const recency = recencyScore(c.lastSeenAt, nowMs);
-  // Use weighted additive formula: importance dominates, recency is a minor boost
-  // This preserves high-importance items even when they haven't been seen recently
-  return importance * 0.7 + recency * 0.3;
-}
-
-function compareProfileCandidates(
-  left: ProfileCandidate,
-  right: ProfileCandidate,
-  nowMs: number,
-  /** When set, entries matching this scope sort before 'default' entries. */
-  preferredScopeId?: string,
-): number {
-  // Scoped entries beat default fallback entries so local overrides win deduplication
-  if (preferredScopeId !== undefined) {
-    const leftIsPreferred = left.scopeId === preferredScopeId;
-    const rightIsPreferred = right.scopeId === preferredScopeId;
-    if (leftIsPreferred && !rightIsPreferred) return -1;
-    if (!leftIsPreferred && rightIsPreferred) return 1;
-  }
-
-  const trustDelta =
-    (TRUST_RANK[right.verificationState] ?? 0) -
-    (TRUST_RANK[left.verificationState] ?? 0);
-  if (trustDelta !== 0) return trustDelta;
-
-  const scoreDelta =
-    candidateRankScore(right, nowMs) - candidateRankScore(left, nowMs);
-  if (scoreDelta !== 0) return scoreDelta;
-
-  const confidenceDelta = right.confidence - left.confidence;
-  if (confidenceDelta !== 0) return confidenceDelta;
-
-  return right.firstSeenAt - left.firstSeenAt;
-}
-
-function normalizeWhitespace(input: string, maxLength: number): string {
-  return input.replace(/\s+/g, " ").trim().slice(0, maxLength);
-}
-
-function renderProfileText(lines: string[]): string {
-  if (lines.length === 0) return "";
-  return ["<dynamic-user-profile>", ...lines, "</dynamic-user-profile>"].join(
-    "\n",
-  );
-}

package/src/memory/recall-cache.ts

@@ -1,117 +0,0 @@
-import { createHash } from "crypto";
-
-import type {
-  MemoryRecallOptions,
-  MemoryRecallResult,
-} from "./search/types.js";
-
-/**
- * In-memory cache for memory recall results.
- *
- * The full retrieval pipeline (FTS5 + Qdrant + entity graph + RRF merge) is
- * expensive. When the same query is issued multiple turns in a row (common
- * when the conversation context hasn't changed), we can serve the cached
- * result instantly.
- *
- * Invalidation: a monotonic version counter is bumped whenever new memory
- * is indexed (segments, items, embeddings). Cache entries are only valid
- * when their version matches the current global version.
- */
-
-interface CacheEntry {
-  version: number;
-  createdAt: number;
-  result: MemoryRecallResult;
-}
-
-const MAX_ENTRIES = 32;
-const TTL_MS = 60_000; // 60 seconds
-
-let _version = 0;
-const _cache = new Map<string, CacheEntry>();
-
-/** Bump the global memory version, invalidating all cached recall results. */
-export function bumpMemoryVersion(): void {
-  _version++;
-}
-
-/** Return the current memory version (for snapshot-based staleness checks). */
-export function getMemoryVersion(): number {
-  return _version;
-}
-
-/** Build a deterministic cache key from the recall inputs. */
-function buildCacheKey(
-  query: string,
-  conversationId: string,
-  options?: MemoryRecallOptions,
-  configFingerprint?: string,
-): string {
-  const parts = [
-    query,
-    conversationId,
-    options?.scopeId ?? "",
-    options?.scopePolicyOverride
-      ? `${options.scopePolicyOverride.scopeId}:${options.scopePolicyOverride.fallbackToDefault}`
-      : "",
-    options?.excludeMessageIds
-      ? [...options.excludeMessageIds].sort().join(",")
-      : "",
-    options?.maxInjectTokensOverride != null
-      ? String(options.maxInjectTokensOverride)
-      : "",
-    configFingerprint ?? "",
-  ];
-  return createHash("sha256").update(parts.join("\0")).digest("hex");
-}
-
-/** Look up a cached recall result. Returns undefined on miss or stale entry. */
-export function getCachedRecall(
-  query: string,
-  conversationId: string,
-  options?: MemoryRecallOptions,
-  configFingerprint?: string,
-): MemoryRecallResult | undefined {
-  const key = buildCacheKey(query, conversationId, options, configFingerprint);
-  const entry = _cache.get(key);
-  if (!entry) return undefined;
-  if (entry.version !== _version || Date.now() - entry.createdAt > TTL_MS) {
-    _cache.delete(key);
-    return undefined;
-  }
-  // Move to end of Map iteration order so it's treated as most-recently-used
-  _cache.delete(key);
-  _cache.set(key, entry);
-  return entry.result;
-}
-
-/**
- * Store a recall result in the cache. Evicts least-recently-used entries when full.
- *
- * When `snapshotVersion` is provided, the entry is only stored if the
- * snapshot still matches the current global version — this prevents a
- * stale result from being cached under a version that was bumped while
- * the retrieval pipeline was in flight.
- */
-export function setCachedRecall(
-  query: string,
-  conversationId: string,
-  options: MemoryRecallOptions | undefined,
-  result: MemoryRecallResult,
-  snapshotVersion?: number,
-  configFingerprint?: string,
-): void {
-  // If a snapshot version was provided, only cache when it still matches
-  // the current version — otherwise the result may be stale.
-  if (snapshotVersion !== undefined && snapshotVersion !== _version) return;
-
-  const key = buildCacheKey(query, conversationId, options, configFingerprint);
-
-  // Evict oldest entries if at capacity
-  if (_cache.size >= MAX_ENTRIES && !_cache.has(key)) {
-    const oldest = _cache.keys().next().value;
-    if (oldest !== undefined) _cache.delete(oldest);
-  }
-
-  _cache.set(key, { version: _version, createdAt: Date.now(), result });
-}