@vellumai/assistant 0.4.49 → 0.4.51

package/src/memory/search/lexical.ts

@@ -1,143 +1,10 @@
 import { and, desc, eq, inArray, notInArray } from "drizzle-orm";
 
-import { getLogger } from "../../util/logger.js";
-import { getDb, rawAll } from "../db.js";
+import { getDb } from "../db.js";
 import { memorySegments } from "../schema.js";
 import { computeRecencyScore } from "./ranking.js";
 import type { Candidate, CandidateType } from "./types.js";
 
-const log = getLogger("memory-retriever");
-
-// Threshold beyond which a raw SQL query is considered slow and logged as a warning.
-const SLOW_QUERY_MS = 2000;
-
-/**
- * Execute a synchronous raw SQL query with timing. Logs a warning if the
- * query exceeds SLOW_QUERY_MS. Since bun:sqlite queries are synchronous,
- * we can't abort them mid-execution, but we can detect and report slowness.
- */
-function timedRawQuery<T>(label: string, fn: () => T[]): T[] {
-  const start = performance.now();
-  const result = fn();
-  const elapsed = performance.now() - start;
-  if (elapsed >= SLOW_QUERY_MS) {
-    log.warn(
-      { label, elapsedMs: Math.round(elapsed) },
-      "Raw SQL query exceeded slow threshold",
-    );
-  }
-  return result;
-}
-
-interface FtsRow {
-  segment_id: string;
-  message_id: string;
-  text: string;
-  created_at: number;
-  rank: number;
-}
-
-export function lexicalSearch(
-  query: string,
-  limit: number,
-  excludedMessageIds: string[] = [],
-  scopeIds?: string[],
-  expandedQuery?: string,
-): Candidate[] {
-  const trimmed = query.trim();
-  if (trimmed.length === 0 || limit <= 0) return [];
-  // When an expanded query is provided (e.g. from query expansion in degraded
-  // mode), use it instead of building one from the raw conversational query.
-  const matchQuery = expandedQuery ?? buildFtsMatchQuery(trimmed);
-  if (!matchQuery) return [];
-  const excluded = new Set(excludedMessageIds);
-  const scopeClause = scopeIds
-    ? ` AND s.scope_id IN (${scopeIds.map(() => "?").join(",")})`
-    : "";
-
-  // Adaptive overfetch: when exclusions are present, double the SQL LIMIT until we
-  // collect `limit` visible rows or the DB has no more matching results to offer.
-  // This handles dense exclusion sets without wasteful fixed-ratio over-fetching.
-  const MAX_FETCH_MULTIPLIER = 8;
-  let queryLimit = excluded.size > 0 ? Math.max(limit * 2, limit + 24) : limit;
-  let visibleRows: FtsRow[] = [];
-
-  while (true) {
-    let rows: FtsRow[] = [];
-    try {
-      rows = timedRawQuery("lexicalSearch", () =>
-        rawAll<FtsRow>(
-          `
-          SELECT
-            f.segment_id AS segment_id,
-            s.message_id AS message_id,
-            s.text AS text,
-            s.created_at AS created_at,
-            bm25(memory_segment_fts) AS rank
-          FROM memory_segment_fts f
-          JOIN memory_segments s ON s.id = f.segment_id
-          WHERE memory_segment_fts MATCH ?${scopeClause}
-          ORDER BY rank
-          LIMIT ?
-        `,
-          matchQuery,
-          ...(scopeIds ?? []),
-          queryLimit,
-        ),
-      );
-    } catch (err) {
-      log.warn(
-        { err, query: truncate(trimmed, 80) },
-        "Memory lexical search query parse failed",
-      );
-      return [];
-    }
-
-    visibleRows =
-      excluded.size > 0
-        ? rows.filter((row) => !excluded.has(row.message_id))
-        : rows;
-
-    // Stop when we have enough rows, the DB returned fewer than requested
-    // (no more results exist), or we've reached the safety cap.
-    if (
-      visibleRows.length >= limit ||
-      rows.length < queryLimit ||
-      queryLimit >= limit * MAX_FETCH_MULTIPLIER
-    ) {
-      break;
-    }
-    queryLimit = Math.min(queryLimit * 2, limit * MAX_FETCH_MULTIPLIER);
-  }
-
-  visibleRows = visibleRows.slice(0, limit);
-
-  const finiteRanks = visibleRows
-    .map((row) => row.rank)
-    .filter((rank) => Number.isFinite(rank));
-  const minRank = finiteRanks.length > 0 ? Math.min(...finiteRanks) : 0;
-  const maxRank = finiteRanks.length > 0 ? Math.max(...finiteRanks) : 0;
-
-  return visibleRows.map((row) => {
-    const lexical = lexicalRankToScore(row.rank, minRank, maxRank);
-    return {
-      key: `segment:${row.segment_id}`,
-      type: "segment" as CandidateType,
-      id: row.segment_id,
-      source: "lexical",
-      text: row.text,
-      kind: "segment",
-      confidence: 0.55,
-      importance: 0.5,
-      createdAt: row.created_at,
-      lexical,
-      semantic: 0,
-      recency: computeRecencyScore(row.created_at),
-      finalScore: 0,
-    };
-  });
-}
-
 export function recencySearch(
   conversationId: string,
   limit: number,
@@ -172,128 +39,8 @@ export function recencySearch(
     confidence: 0.55,
     importance: 0.5,
     createdAt: row.createdAt,
-    lexical: 0,
     semantic: 0,
     recency: computeRecencyScore(row.createdAt),
     finalScore: 0,
   }));
 }
-
-/**
- * Direct search over memory_items table by subject and statement text.
- * Supplements FTS-based lexical search with LIKE-based matching on items.
- */
-export function directItemSearch(
-  query: string,
-  limit: number,
-  scopeIds?: string[],
-): Candidate[] {
-  const tokens = [
-    ...new Set(
-      query
-        .toLowerCase()
-        .split(/[^a-z0-9_.-]+/g)
-        .filter((t) => t.length >= 2),
-    ),
-  ];
-  if (tokens.length === 0) return [];
-
-  const likeClauses = tokens.map(
-    () => `(LOWER(subject) LIKE ? OR LOWER(statement) LIKE ?)`,
-  );
-  const likeParams = tokens.flatMap((t) => {
-    const pattern = `%${escapeLikeWildcards(t)}%`;
-    return [pattern, pattern];
-  });
-  const scopeClause = scopeIds
-    ? ` AND scope_id IN (${scopeIds.map(() => "?").join(",")})`
-    : "";
-  const sqlQuery = `
-    SELECT id, kind, subject, statement, status, confidence, importance, first_seen_at, last_seen_at
-    FROM memory_items
-    WHERE status = 'active' AND invalid_at IS NULL AND (${likeClauses.join(
-      " OR ",
-    )})${scopeClause}
-    ORDER BY last_seen_at DESC
-    LIMIT ?
-  `;
-
-  interface ItemRow {
-    id: string;
-    kind: string;
-    subject: string;
-    statement: string;
-    confidence: number;
-    importance: number | null;
-    first_seen_at: number;
-    last_seen_at: number;
-  }
-
-  let rows: ItemRow[] = [];
-  try {
-    rows = timedRawQuery("directItemSearch", () =>
-      rawAll<ItemRow>(sqlQuery, ...likeParams, ...(scopeIds ?? []), limit),
-    );
-  } catch {
-    return [];
-  }
-
-  return rows.map((row) => {
-    // Compute lexical score based on token match coverage: fraction of query
-    // tokens that appear in subject or statement. Direct items are keyword
-    // matches so this score reflects query-match relevance (unlike confidence,
-    // which reflects extraction certainty).
-    const textLower = `${row.subject} ${row.statement}`.toLowerCase();
-    const matchedTokens = tokens.filter((t) => textLower.includes(t)).length;
-    const lexical = tokens.length > 0 ? matchedTokens / tokens.length : 0;
-
-    return {
-      key: `item:${row.id}`,
-      type: "item" as CandidateType,
-      id: row.id,
-      source: "item_direct",
-      text: `${row.subject}: ${row.statement}`,
-      kind: row.kind,
-      confidence: row.confidence,
-      importance: row.importance ?? 0.5,
-      createdAt: row.last_seen_at,
-      lexical,
-      semantic: 0,
-      recency: computeRecencyScore(row.last_seen_at),
-      finalScore: 0,
-    };
-  });
-}
-
-export function lexicalRankToScore(
-  rank: number,
-  minRank: number,
-  maxRank: number,
-): number {
-  if (!Number.isFinite(rank)) return 0;
-  if (!Number.isFinite(minRank) || !Number.isFinite(maxRank)) return 0;
-  const span = maxRank - minRank;
-  if (span <= 0) return 1;
-  // Lower BM25 rank is better in FTS5; normalize to [0,1] where 1 is best.
-  return (maxRank - rank) / span;
-}
-
-export function buildFtsMatchQuery(text: string): string | null {
-  const tokens = text
-    .toLowerCase()
-    .split(/[^a-z0-9_]+/g)
-    .map((token) => token.trim())
-    .filter((token) => token.length >= 2);
-  if (tokens.length === 0) return null;
-  const unique = [...new Set(tokens)].slice(0, 24);
-  return unique.map((token) => `"${token.replace(/"/g, '""')}"`).join(" OR ");
-}
-
-export function escapeLikeWildcards(s: string): string {
-  return s.replace(/%/g, "").replace(/_/g, "");
-}
-
-function truncate(text: string, max: number): string {
-  if (text.length <= max) return text;
-  return `${text.slice(0, max - 3)}...`;
-}