@optave/codegraph 3.1.3 → 3.1.5

package/src/domain/search/search/cli-formatter.js

@@ -0,0 +1,151 @@
+import { warn } from '../../../infrastructure/logger.js';
+import { hybridSearchData } from './hybrid.js';
+import { ftsSearchData } from './keyword.js';
+import { multiSearchData, searchData } from './semantic.js';
+
+/**
+ * Search with mode support — CLI wrapper with multi-query detection.
+ * Modes: 'hybrid' (default), 'semantic', 'keyword'
+ */
+export async function search(query, customDbPath, opts = {}) {
+  const mode = opts.mode || 'hybrid';
+
+  // Split by semicolons, trim, filter empties
+  const queries = query
+    .split(';')
+    .map((q) => q.trim())
+    .filter((q) => q.length > 0);
+
+  const kindIcon = (kind) => (kind === 'function' ? 'f' : kind === 'class' ? '*' : 'o');
+
+  // ─── Keyword-only mode ──────────────────────────────────────────────
+  if (mode === 'keyword') {
+    const singleQuery = queries.length === 1 ? queries[0] : query;
+    const data = ftsSearchData(singleQuery, customDbPath, opts);
+    if (!data) {
+      console.log('No FTS5 index found. Run `codegraph embed` to build the keyword index.');
+      return;
+    }
+
+    if (opts.json) {
+      console.log(JSON.stringify(data, null, 2));
+      return;
+    }
+
+    console.log(`\nKeyword search: "${singleQuery}" (BM25)\n`);
+    if (data.results.length === 0) {
+      console.log('  No results found.');
+    } else {
+      for (const r of data.results) {
+        console.log(
+          `  BM25 ${r.bm25Score.toFixed(2)}  ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`,
+        );
+      }
+    }
+    console.log(`\n  ${data.results.length} results shown\n`);
+    return;
+  }
+
+  // ─── Semantic-only mode ─────────────────────────────────────────────
+  if (mode === 'semantic') {
+    if (queries.length <= 1) {
+      const singleQuery = queries[0] || query;
+      const data = await searchData(singleQuery, customDbPath, opts);
+      if (!data) return;
+
+      if (opts.json) {
+        console.log(JSON.stringify(data, null, 2));
+        return;
+      }
+
+      console.log(`\nSemantic search: "${singleQuery}"\n`);
+      if (data.results.length === 0) {
+        console.log('  No results above threshold.');
+      } else {
+        for (const r of data.results) {
+          const bar = '#'.repeat(Math.round(r.similarity * 20));
+          console.log(`  ${(r.similarity * 100).toFixed(1)}% ${bar}`);
+          console.log(`    ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`);
+        }
+      }
+      console.log(`\n  ${data.results.length} results shown\n`);
+    } else {
+      const data = await multiSearchData(queries, customDbPath, opts);
+      if (!data) return;
+
+      if (opts.json) {
+        console.log(JSON.stringify(data, null, 2));
+        return;
+      }
+
+      console.log(`\nMulti-query semantic search (RRF, k=${opts.rrfK || 60}):`);
+      for (let i = 0; i < queries.length; i++) console.log(`  [${i + 1}] "${queries[i]}"`);
+      console.log();
+      if (data.results.length === 0) {
+        console.log('  No results above threshold.');
+      } else {
+        for (const r of data.results) {
+          console.log(
+            `  RRF ${r.rrf.toFixed(4)}  ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`,
+          );
+          for (const qs of r.queryScores) {
+            const bar = '#'.repeat(Math.round(qs.similarity * 20));
+            console.log(
+              `    [${queries.indexOf(qs.query) + 1}] ${(qs.similarity * 100).toFixed(1)}% ${bar} (rank ${qs.rank})`,
+            );
+          }
+        }
+      }
+      console.log(`\n  ${data.results.length} results shown\n`);
+    }
+    return;
+  }
+
+  // ─── Hybrid mode (default) ──────────────────────────────────────────
+  const data = await hybridSearchData(query, customDbPath, opts);
+
+  if (!data) {
+    // No FTS5 index — fall back to semantic-only
+    warn(
+      'FTS5 index not found — using semantic search only. Re-run `codegraph embed` to enable hybrid mode.',
+    );
+    return search(query, customDbPath, { ...opts, mode: 'semantic' });
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify(data, null, 2));
+    return;
+  }
+
+  const rrfK = opts.rrfK || 60;
+  if (queries.length <= 1) {
+    const singleQuery = queries[0] || query;
+    console.log(`\nHybrid search: "${singleQuery}" (BM25 + semantic, RRF k=${rrfK})\n`);
+  } else {
+    console.log(`\nHybrid multi-query search (BM25 + semantic, RRF k=${rrfK}):`);
+    for (let i = 0; i < queries.length; i++) console.log(`  [${i + 1}] "${queries[i]}"`);
+    console.log();
+  }
+
+  if (data.results.length === 0) {
+    console.log('  No results found.');
+  } else {
+    for (const r of data.results) {
+      console.log(
+        `  RRF ${r.rrf.toFixed(4)}  ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`,
+      );
+      const parts = [];
+      if (r.bm25Rank != null) {
+        parts.push(`BM25: rank ${r.bm25Rank} (score ${r.bm25Score.toFixed(2)})`);
+      }
+      if (r.semanticRank != null) {
+        parts.push(`Semantic: rank ${r.semanticRank} (${(r.similarity * 100).toFixed(1)}%)`);
+      }
+      if (parts.length > 0) {
+        console.log(`    ${parts.join('  |  ')}`);
+      }
+    }
+  }
+
+  console.log(`\n  ${data.results.length} results shown\n`);
+}

package/src/domain/search/search/filters.js

@@ -0,0 +1,46 @@
+/**
+ * Match a file path against a glob pattern.
+ * Supports *, **, and ? wildcards. Zero dependencies.
+ */
+export function globMatch(filePath, pattern) {
+  // Normalize separators to forward slashes
+  const normalized = filePath.replace(/\\/g, '/');
+  // Escape regex specials except glob chars
+  let regex = pattern.replace(/\\/g, '/').replace(/[.+^${}()|[\]\\]/g, '\\$&');
+  // Replace ** first (matches any path segment), then * and ?
+  regex = regex.replace(/\*\*/g, '\0');
+  regex = regex.replace(/\*/g, '[^/]*');
+  regex = regex.replace(/\0/g, '.*');
+  regex = regex.replace(/\?/g, '[^/]');
+  try {
+    return new RegExp(`^${regex}$`).test(normalized);
+  } catch {
+    // Malformed pattern — fall back to substring match
+    return normalized.includes(pattern);
+  }
+}
+
+const TEST_PATTERN = /\.(test|spec)\.|__test__|__tests__|\.stories\./;
+
+/**
+ * Apply post-query filters (glob pattern, noTests) to a set of rows.
+ * Mutates nothing — returns a new filtered array.
+ * @param {Array} rows - Rows with at least a `file` property
+ * @param {object} opts
+ * @param {string} [opts.filePattern] - Glob pattern (only applied if it contains glob chars)
+ * @param {boolean} [opts.noTests] - Exclude test/spec files
+ * @param {boolean} [opts.isGlob] - Pre-computed: does filePattern contain glob chars?
+ * @returns {Array}
+ */
+export function applyFilters(rows, opts = {}) {
+  let filtered = rows;
+  const isGlob =
+    opts.isGlob !== undefined ? opts.isGlob : opts.filePattern && /[*?[\]]/.test(opts.filePattern);
+  if (isGlob) {
+    filtered = filtered.filter((row) => globMatch(row.file, opts.filePattern));
+  }
+  if (opts.noTests) {
+    filtered = filtered.filter((row) => !TEST_PATTERN.test(row.file));
+  }
+  return filtered;
+}

package/src/domain/search/search/hybrid.js

@@ -0,0 +1,121 @@
+import { openReadonlyOrFail } from '../../../db/index.js';
+import { hasFtsIndex } from '../stores/fts5.js';
+import { ftsSearchData } from './keyword.js';
+import { searchData } from './semantic.js';
+
+/**
+ * Hybrid BM25 + semantic search with RRF fusion.
+ * Returns { results: [{ name, kind, file, line, rrf, bm25Score, bm25Rank, similarity, semanticRank }] }
+ * or null if no FTS5 index (caller should fall back to semantic-only).
+ */
+export async function hybridSearchData(query, customDbPath, opts = {}) {
+  const limit = opts.limit || 15;
+  const k = opts.rrfK || 60;
+  const topK = (opts.limit || 15) * 5;
+
+  // Split semicolons for multi-query support
+  const queries =
+    typeof query === 'string'
+      ? query
+          .split(';')
+          .map((q) => q.trim())
+          .filter((q) => q.length > 0)
+      : [query];
+
+  // Check FTS5 availability first (sync, cheap)
+  const checkDb = openReadonlyOrFail(customDbPath);
+  const ftsAvailable = hasFtsIndex(checkDb);
+  checkDb.close();
+  if (!ftsAvailable) return null;
+
+  // Collect ranked lists: for each query, one BM25 list + one semantic list
+  const rankedLists = [];
+
+  for (const q of queries) {
+    // BM25 ranked list (sync)
+    const bm25Data = ftsSearchData(q, customDbPath, { ...opts, limit: topK });
+    if (bm25Data?.results) {
+      rankedLists.push(
+        bm25Data.results.map((r, idx) => ({
+          key: `${r.name}:${r.file}:${r.line}`,
+          rank: idx + 1,
+          source: 'bm25',
+          ...r,
+        })),
+      );
+    }
+
+    // Semantic ranked list (async)
+    const semData = await searchData(q, customDbPath, {
+      ...opts,
+      limit: topK,
+      minScore: opts.minScore || 0.2,
+    });
+    if (semData?.results) {
+      rankedLists.push(
+        semData.results.map((r, idx) => ({
+          key: `${r.name}:${r.file}:${r.line}`,
+          rank: idx + 1,
+          source: 'semantic',
+          ...r,
+        })),
+      );
+    }
+  }
+
+  // RRF fusion across all ranked lists
+  const fusionMap = new Map();
+  for (const list of rankedLists) {
+    for (const item of list) {
+      if (!fusionMap.has(item.key)) {
+        fusionMap.set(item.key, {
+          name: item.name,
+          kind: item.kind,
+          file: item.file,
+          line: item.line,
+          endLine: item.endLine ?? null,
+          role: item.role ?? null,
+          fileHash: item.fileHash ?? null,
+          rrfScore: 0,
+          bm25Score: null,
+          bm25Rank: null,
+          similarity: null,
+          semanticRank: null,
+        });
+      }
+      const entry = fusionMap.get(item.key);
+      entry.rrfScore += 1 / (k + item.rank);
+      if (item.source === 'bm25') {
+        if (entry.bm25Rank === null || item.rank < entry.bm25Rank) {
+          entry.bm25Score = item.bm25Score;
+          entry.bm25Rank = item.rank;
+        }
+      } else {
+        if (entry.semanticRank === null || item.rank < entry.semanticRank) {
+          entry.similarity = item.similarity;
+          entry.semanticRank = item.rank;
+        }
+      }
+    }
+  }
+
+  const results = [...fusionMap.values()]
+    .sort((a, b) => b.rrfScore - a.rrfScore)
+    .slice(0, limit)
+    .map((e) => ({
+      name: e.name,
+      kind: e.kind,
+      file: e.file,
+      line: e.line,
+      endLine: e.endLine,
+      role: e.role,
+      fileHash: e.fileHash,
+      rrf: e.rrfScore,
+      bm25Score: e.bm25Score,
+      bm25Rank: e.bm25Rank,
+      similarity: e.similarity,
+      semanticRank: e.semanticRank,
+    }));
+
+  return { results };
+}

package/src/domain/search/search/keyword.js

@@ -0,0 +1,68 @@
+import { openReadonlyOrFail } from '../../../db/index.js';
+import { normalizeSymbol } from '../../queries.js';
+import { hasFtsIndex, sanitizeFtsQuery } from '../stores/fts5.js';
+import { applyFilters } from './filters.js';
+
+/**
+ * BM25 keyword search via FTS5.
+ * Returns { results: [{ name, kind, file, line, bm25Score }] } or null if no FTS5 index.
+ */
+export function ftsSearchData(query, customDbPath, opts = {}) {
+  const limit = opts.limit || 15;
+
+  const db = openReadonlyOrFail(customDbPath);
+
+  try {
+    if (!hasFtsIndex(db)) {
+      return null;
+    }
+
+    const ftsQuery = sanitizeFtsQuery(query);
+    if (!ftsQuery) {
+      return { results: [] };
+    }
+
+    let sql = `
+      SELECT f.rowid AS node_id, rank AS bm25_score,
+             n.name, n.kind, n.file, n.line, n.end_line, n.role
+      FROM fts_index f
+      JOIN nodes n ON f.rowid = n.id
+      WHERE fts_index MATCH ?
+    `;
+    const params = [ftsQuery];
+
+    if (opts.kind) {
+      sql += ' AND n.kind = ?';
+      params.push(opts.kind);
+    }
+
+    const isGlob = opts.filePattern && /[*?[\]]/.test(opts.filePattern);
+    if (opts.filePattern && !isGlob) {
+      sql += ' AND n.file LIKE ?';
+      params.push(`%${opts.filePattern}%`);
+    }
+
+    sql += ' ORDER BY rank LIMIT ?';
+    params.push(limit * 5); // fetch generous set for post-filtering
+
+    let rows;
+    try {
+      rows = db.prepare(sql).all(...params);
+    } catch {
+      // Invalid FTS5 query syntax — return empty
+      return { results: [] };
+    }
+
+    rows = applyFilters(rows, { ...opts, isGlob });
+
+    const hc = new Map();
+    const results = rows.slice(0, limit).map((row) => ({
+      ...normalizeSymbol(row, db, hc),
+      bm25Score: -row.bm25_score, // FTS5 rank is negative; negate for display
+    }));
+
+    return { results };
+  } finally {
+    db.close();
+  }
+}

package/src/domain/search/search/prepare.js

@@ -0,0 +1,66 @@
+import { openReadonlyOrFail } from '../../../db/index.js';
+import { getEmbeddingCount, getEmbeddingMeta } from '../../../db/repository/embeddings.js';
+import { MODELS } from '../models.js';
+import { applyFilters } from './filters.js';
+
+/**
+ * Shared setup for search functions: opens DB, validates embeddings/model, loads rows.
+ * Returns { db, rows, modelKey, storedDim } or null on failure (prints error).
+ * On null return, the DB is closed. On exception, the DB is also closed
+ * (callers only need to close DB from the returned object on the happy path).
+ */
+export function prepareSearch(customDbPath, opts = {}) {
+  const db = openReadonlyOrFail(customDbPath);
+
+  try {
+    const count = getEmbeddingCount(db);
+    if (count === 0) {
+      console.log('No embeddings found. Run `codegraph embed` first.');
+      db.close();
+      return null;
+    }
+
+    const storedModel = getEmbeddingMeta(db, 'model') || null;
+    const dimStr = getEmbeddingMeta(db, 'dim');
+    const storedDim = dimStr ? parseInt(dimStr, 10) : null;
+
+    let modelKey = opts.model || null;
+    if (!modelKey && storedModel) {
+      for (const [key, config] of Object.entries(MODELS)) {
+        if (config.name === storedModel) {
+          modelKey = key;
+          break;
+        }
+      }
+    }
+
+    // Pre-filter: allow filtering by kind or file pattern to reduce search space
+    const isGlob = opts.filePattern && /[*?[\]]/.test(opts.filePattern);
+    let sql = `
+    SELECT e.node_id, e.vector, e.text_preview, n.name, n.kind, n.file, n.line, n.end_line, n.role
+    FROM embeddings e
+    JOIN nodes n ON e.node_id = n.id
+  `;
+    const params = [];
+    const conditions = [];
+    if (opts.kind) {
+      conditions.push('n.kind = ?');
+      params.push(opts.kind);
+    }
+    if (opts.filePattern && !isGlob) {
+      conditions.push('n.file LIKE ?');
+      params.push(`%${opts.filePattern}%`);
+    }
+    if (conditions.length > 0) {
+      sql += ` WHERE ${conditions.join(' AND ')}`;
+    }
+
+    let rows = db.prepare(sql).all(...params);
+    rows = applyFilters(rows, { ...opts, isGlob });
+
+    return { db, rows, modelKey, storedDim };
+  } catch (err) {
+    db.close();
+    throw err;
+  }
+}

package/src/domain/search/search/semantic.js

@@ -0,0 +1,145 @@
+import { warn } from '../../../infrastructure/logger.js';
+import { normalizeSymbol } from '../../queries.js';
+import { embed } from '../models.js';
+import { cosineSim } from '../stores/sqlite-blob.js';
+import { prepareSearch } from './prepare.js';
+
+/**
+ * Single-query semantic search — returns data instead of printing.
+ * Returns { results: [{ name, kind, file, line, similarity }] } or null on failure.
+ */
+export async function searchData(query, customDbPath, opts = {}) {
+  const limit = opts.limit || 15;
+  const minScore = opts.minScore || 0.2;
+
+  const prepared = prepareSearch(customDbPath, opts);
+  if (!prepared) return null;
+  const { db, rows, modelKey, storedDim } = prepared;
+
+  try {
+    const {
+      vectors: [queryVec],
+      dim,
+    } = await embed([query], modelKey);
+
+    if (storedDim && dim !== storedDim) {
+      console.log(
+        `Warning: query model dimension (${dim}) doesn't match stored embeddings (${storedDim}).`,
+      );
+      console.log(`  Re-run \`codegraph embed\` with the same model, or use --model to match.`);
+      return null;
+    }
+
+    const hc = new Map();
+    const results = [];
+    for (const row of rows) {
+      const vec = new Float32Array(new Uint8Array(row.vector).buffer);
+      const sim = cosineSim(queryVec, vec);
+
+      if (sim >= minScore) {
+        results.push({
+          ...normalizeSymbol(row, db, hc),
+          similarity: sim,
+        });
+      }
+    }
+
+    results.sort((a, b) => b.similarity - a.similarity);
+    return { results: results.slice(0, limit) };
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Multi-query semantic search with Reciprocal Rank Fusion (RRF).
+ * Returns { results: [{ name, kind, file, line, rrf, queryScores }] } or null on failure.
+ */
+export async function multiSearchData(queries, customDbPath, opts = {}) {
+  const limit = opts.limit || 15;
+  const minScore = opts.minScore || 0.2;
+  const k = opts.rrfK || 60;
+
+  const prepared = prepareSearch(customDbPath, opts);
+  if (!prepared) return null;
+  const { db, rows, modelKey, storedDim } = prepared;
+
+  try {
+    const { vectors: queryVecs, dim } = await embed(queries, modelKey);
+
+    // Warn about similar queries that may bias RRF results
+    const SIMILARITY_WARN_THRESHOLD = 0.85;
+    for (let i = 0; i < queryVecs.length; i++) {
+      for (let j = i + 1; j < queryVecs.length; j++) {
+        const sim = cosineSim(queryVecs[i], queryVecs[j]);
+        if (sim >= SIMILARITY_WARN_THRESHOLD) {
+          warn(
+            `Queries "${queries[i]}" and "${queries[j]}" are very similar ` +
+              `(${(sim * 100).toFixed(0)}% cosine similarity). ` +
+              `This may bias RRF results toward their shared matches. ` +
+              `Consider using more distinct queries.`,
+          );
+        }
+      }
+    }
+
+    if (storedDim && dim !== storedDim) {
+      console.log(
+        `Warning: query model dimension (${dim}) doesn't match stored embeddings (${storedDim}).`,
+      );
+      console.log(`  Re-run \`codegraph embed\` with the same model, or use --model to match.`);
+      return null;
+    }
+
+    // Parse row vectors once
+    const rowVecs = rows.map((row) => new Float32Array(new Uint8Array(row.vector).buffer));
+
+    // For each query: compute similarities, filter by minScore, rank
+    const perQueryRanked = queries.map((_query, qi) => {
+      const scored = [];
+      for (let ri = 0; ri < rows.length; ri++) {
+        const sim = cosineSim(queryVecs[qi], rowVecs[ri]);
+        if (sim >= minScore) {
+          scored.push({ rowIndex: ri, similarity: sim });
+        }
+      }
+      scored.sort((a, b) => b.similarity - a.similarity);
+      // Assign 1-indexed ranks
+      return scored.map((item, rank) => ({ ...item, rank: rank + 1 }));
+    });
+
+    // Fuse results using RRF: for each unique row, sum 1/(k + rank_i) across queries
+    const fusionMap = new Map(); // rowIndex -> { rrfScore, queryScores[] }
+    for (let qi = 0; qi < queries.length; qi++) {
+      for (const item of perQueryRanked[qi]) {
+        if (!fusionMap.has(item.rowIndex)) {
+          fusionMap.set(item.rowIndex, { rrfScore: 0, queryScores: [] });
+        }
+        const entry = fusionMap.get(item.rowIndex);
+        entry.rrfScore += 1 / (k + item.rank);
+        entry.queryScores.push({
+          query: queries[qi],
+          similarity: item.similarity,
+          rank: item.rank,
+        });
+      }
+    }
+
+    // Build results sorted by RRF score
+    const hc = new Map();
+    const results = [];
+    for (const [rowIndex, entry] of fusionMap) {
+      const row = rows[rowIndex];
+      results.push({
+        ...normalizeSymbol(row, db, hc),
+        rrf: entry.rrfScore,
+        queryScores: entry.queryScores,
+      });
+    }
+
+    results.sort((a, b) => b.rrf - a.rrf);
+    return { results: results.slice(0, limit) };
+  } finally {
+    db.close();
+  }
+}

package/src/domain/search/stores/fts5.js

@@ -0,0 +1,27 @@
+/**
+ * Sanitize a user query for FTS5 MATCH syntax.
+ * Wraps each token as an implicit OR and escapes special FTS5 characters.
+ */
+export function sanitizeFtsQuery(query) {
+  // Remove FTS5 special chars that could cause syntax errors
+  const cleaned = query.replace(/[*"():^{}~<>]/g, ' ').trim();
+  if (!cleaned) return null;
+  // Split into tokens, wrap with OR for multi-token queries
+  const tokens = cleaned.split(/\s+/).filter((t) => t.length > 0);
+  if (tokens.length === 0) return null;
+  if (tokens.length === 1) return `"${tokens[0]}"`;
+  return tokens.map((t) => `"${t}"`).join(' OR ');
+}
+
+/**
+ * Check if the FTS5 index exists in the database.
+ * Returns true if fts_index table exists and has rows, false otherwise.
+ */
+export function hasFtsIndex(db) {
+  try {
+    const row = db.prepare('SELECT COUNT(*) as c FROM fts_index').get();
+    return row.c > 0;
+  } catch {
+    return false;
+  }
+}

package/src/domain/search/stores/sqlite-blob.js

@@ -0,0 +1,24 @@
+/**
+ * @typedef {object} VectorStore
+ * @property {(queryVec: Float32Array, rows: Array<{vector: Buffer}>) => Array<{index: number, score: number}>} search
+ *   Score every row against a query vector and return scored indices.
+ *
+ * Future implementations (e.g. HNSW via `hnsw.js`) implement this same shape
+ * for approximate nearest-neighbor search.
+ */
+
+/**
+ * Cosine similarity between two Float32Arrays.
+ */
+export function cosineSim(a, b) {
+  let dot = 0,
+    normA = 0,
+    normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}

package/src/domain/search/strategies/source.js

@@ -0,0 +1,14 @@
+import { splitIdentifier } from './text-utils.js';
+
+/**
+ * Build raw source-code text for a symbol (original strategy).
+ */
+export function buildSourceText(node, file, lines) {
+  const startLine = Math.max(0, node.line - 1);
+  const endLine = node.end_line
+    ? Math.min(lines.length, node.end_line)
+    : Math.min(lines.length, startLine + 15);
+  const context = lines.slice(startLine, endLine).join('\n');
+  const readable = splitIdentifier(node.name);
+  return `${node.kind} ${node.name} (${readable}) in ${file}\n${context}`;
+}