optave-codegraph 3.13.0

package/src/domain/search/search/cli-formatter.ts

@@ -0,0 +1,179 @@
+import { warn } from '../../../infrastructure/logger.js';
+import { hybridSearchData } from './hybrid.js';
+import { ftsSearchData } from './keyword.js';
+import type { SemanticSearchOpts } from './semantic.js';
+import { multiSearchData, searchData } from './semantic.js';
+
+interface SearchOpts extends SemanticSearchOpts {
+  mode?: 'hybrid' | 'semantic' | 'keyword';
+  json?: boolean;
+  ndjson?: boolean;
+  offset?: number;
+}
+
+const kindIcon = (kind: string): string =>
+  kind === 'function' ? 'f' : kind === 'class' ? '*' : 'o';
+
+function formatKeywordResults(
+  singleQuery: string,
+  customDbPath: string | undefined,
+  opts: SearchOpts,
+): void {
+  const data = ftsSearchData(singleQuery, customDbPath, opts);
+  if (!data) {
+    console.log('No FTS5 index found. Run `codegraph embed` to build the keyword index.');
+    return;
+  }
+  if (opts.json) {
+    console.log(JSON.stringify(data, null, 2));
+    return;
+  }
+  console.log(`\nKeyword search: "${singleQuery}" (BM25)\n`);
+  if (data.results.length === 0) {
+    console.log('  No results found.');
+  } else {
+    for (const r of data.results) {
+      console.log(
+        `  BM25 ${r.bm25Score.toFixed(2)}  ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`,
+      );
+    }
+  }
+  console.log(`\n  ${data.results.length} results shown\n`);
+}
+
+async function formatSemanticSingle(
+  singleQuery: string,
+  customDbPath: string | undefined,
+  opts: SearchOpts,
+): Promise<void> {
+  const data = await searchData(singleQuery, customDbPath, opts);
+  if (!data) return;
+  if (opts.json) {
+    console.log(JSON.stringify(data, null, 2));
+    return;
+  }
+  console.log(`\nSemantic search: "${singleQuery}"\n`);
+  if (data.results.length === 0) {
+    console.log('  No results above threshold.');
+  } else {
+    for (const r of data.results) {
+      const bar = '#'.repeat(Math.round(r.similarity * 20));
+      console.log(`  ${(r.similarity * 100).toFixed(1)}% ${bar}`);
+      console.log(`    ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`);
+    }
+  }
+  console.log(`\n  ${data.results.length} results shown\n`);
+}
+
+async function formatSemanticMulti(
+  queries: string[],
+  customDbPath: string | undefined,
+  opts: SearchOpts,
+): Promise<void> {
+  const data = await multiSearchData(queries, customDbPath, opts);
+  if (!data) return;
+  if (opts.json) {
+    console.log(JSON.stringify(data, null, 2));
+    return;
+  }
+  console.log(`\nMulti-query semantic search (RRF, k=${opts.rrfK || 60}):`);
+  for (let i = 0; i < queries.length; i++) console.log(`  [${i + 1}] "${queries[i]}"`);
+  console.log();
+  if (data.results.length === 0) {
+    console.log('  No results above threshold.');
+  } else {
+    for (const r of data.results) {
+      console.log(
+        `  RRF ${r.rrf.toFixed(4)}  ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`,
+      );
+      for (const qs of r.queryScores) {
+        const bar = '#'.repeat(Math.round(qs.similarity * 20));
+        console.log(
+          `    [${queries.indexOf(qs.query) + 1}] ${(qs.similarity * 100).toFixed(1)}% ${bar} (rank ${qs.rank})`,
+        );
+      }
+    }
+  }
+  console.log(`\n  ${data.results.length} results shown\n`);
+}
+
+function formatHybridResults(
+  queries: string[],
+  query: string,
+  data: { results: any[] },
+  opts: SearchOpts,
+): void {
+  const rrfK = opts.rrfK || 60;
+  if (queries.length <= 1) {
+    const singleQuery = queries[0] || query;
+    console.log(`\nHybrid search: "${singleQuery}" (BM25 + semantic, RRF k=${rrfK})\n`);
+  } else {
+    console.log(`\nHybrid multi-query search (BM25 + semantic, RRF k=${rrfK}):`);
+    for (let i = 0; i < queries.length; i++) console.log(`  [${i + 1}] "${queries[i]}"`);
+    console.log();
+  }
+
+  if (data.results.length === 0) {
+    console.log('  No results found.');
+  } else {
+    for (const r of data.results) {
+      console.log(
+        `  RRF ${r.rrf.toFixed(4)}  ${kindIcon(r.kind)} ${r.name} -- ${r.file}:${r.line}`,
+      );
+      const parts: string[] = [];
+      if (r.bm25Rank != null) {
+        parts.push(`BM25: rank ${r.bm25Rank} (score ${r.bm25Score!.toFixed(2)})`);
+      }
+      if (r.semanticRank != null) {
+        parts.push(`Semantic: rank ${r.semanticRank} (${(r.similarity! * 100).toFixed(1)}%)`);
+      }
+      if (parts.length > 0) {
+        console.log(`    ${parts.join('  |  ')}`);
+      }
+    }
+  }
+
+  console.log(`\n  ${data.results.length} results shown\n`);
+}
+
+export async function search(
+  query: string,
+  customDbPath: string | undefined,
+  opts: SearchOpts = {},
+): Promise<void> {
+  const mode = opts.mode || 'hybrid';
+
+  const queries = query
+    .split(';')
+    .map((q) => q.trim())
+    .filter((q) => q.length > 0);
+
+  if (mode === 'keyword') {
+    const singleQuery = queries.length === 1 ? queries[0]! : query;
+    return formatKeywordResults(singleQuery, customDbPath, opts);
+  }
+
+  if (mode === 'semantic') {
+    if (queries.length <= 1) {
+      return formatSemanticSingle(queries[0] || query, customDbPath, opts);
+    }
+    return formatSemanticMulti(queries, customDbPath, opts);
+  }
+
+  // Hybrid mode (default)
+  const data = await hybridSearchData(query, customDbPath, opts);
+
+  if (!data) {
+    warn(
+      'FTS5 index not found — using semantic search only. Re-run `codegraph embed` to enable hybrid mode.',
+    );
+    return search(query, customDbPath, { ...opts, mode: 'semantic' });
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify(data, null, 2));
+    return;
+  }
+
+  formatHybridResults(queries, query, data, opts);
+}

package/src/domain/search/search/filters.ts

@@ -0,0 +1,38 @@
+export function globMatch(filePath: string, pattern: string): boolean {
+  const normalized = filePath.replace(/\\/g, '/');
+  let regex = pattern.replace(/\\/g, '/').replace(/[.+^${}()|[\]\\]/g, '\\$&');
+  regex = regex.replace(/\*\*/g, '\0');
+  regex = regex.replace(/\*/g, '[^/]*');
+  regex = regex.replace(/\0/g, '.*');
+  regex = regex.replace(/\?/g, '[^/]');
+  try {
+    return new RegExp(`^${regex}$`).test(normalized);
+  } catch {
+    return normalized.includes(pattern);
+  }
+}
+
+const TEST_PATTERN = /\.(test|spec)\.|__test__|__tests__|\.stories\./;
+
+export interface FilterOpts {
+  filePattern?: string | string[];
+  noTests?: boolean;
+}
+
+export function applyFilters<T extends { file: string }>(rows: T[], opts: FilterOpts = {}): T[] {
+  let filtered = rows;
+  const fp = opts.filePattern;
+  const fpArr = Array.isArray(fp) ? fp : fp ? [fp] : [];
+  if (fpArr.length > 0) {
+    filtered = filtered.filter((row) =>
+      fpArr.some((p) => {
+        const patternIsGlob = /[*?[\]]/.test(p);
+        return patternIsGlob ? globMatch(row.file, p) : row.file.includes(p);
+      }),
+    );
+  }
+  if (opts.noTests) {
+    filtered = filtered.filter((row) => !TEST_PATTERN.test(row.file));
+  }
+  return filtered;
+}

package/src/domain/search/search/hybrid.ts

@@ -0,0 +1,197 @@
+import { openReadonlyOrFail } from '../../../db/index.js';
+import { loadConfig } from '../../../infrastructure/config.js';
+import type { BetterSqlite3Database, CodegraphConfig } from '../../../types.js';
+import { hasFtsIndex } from '../stores/fts5.js';
+import { ftsSearchData } from './keyword.js';
+import type { SemanticSearchOpts } from './semantic.js';
+import { searchData } from './semantic.js';
+
+interface HybridResult {
+  name: string;
+  kind: string;
+  file: string;
+  line: number;
+  endLine: number | null;
+  role: string | null;
+  fileHash: string | null;
+  rrf: number;
+  bm25Score: number | null;
+  bm25Rank: number | null;
+  similarity: number | null;
+  semanticRank: number | null;
+}
+
+export interface HybridSearchResult {
+  results: HybridResult[];
+}
+
+interface RankedItem {
+  key: string;
+  rank: number;
+  source: 'bm25' | 'semantic';
+  name: string;
+  kind: string;
+  file: string;
+  line: number;
+  endLine?: number | null;
+  role?: string | null;
+  fileHash?: string | null;
+  bm25Score?: number;
+  similarity?: number;
+}
+
+interface FusionEntry {
+  name: string;
+  kind: string;
+  file: string;
+  line: number;
+  endLine: number | null;
+  role: string | null;
+  fileHash: string | null;
+  rrfScore: number;
+  bm25Score: number | null;
+  bm25Rank: number | null;
+  similarity: number | null;
+  semanticRank: number | null;
+}
+
+/** Parse a semicolon-delimited query string into individual queries. */
+function parseQueries(query: string): string[] {
+  return query
+    .split(';')
+    .map((q) => q.trim())
+    .filter((q) => q.length > 0);
+}
+
+/** Collect BM25 and semantic ranked lists for each query. */
+async function collectRankedLists(
+  queries: string[],
+  customDbPath: string | undefined,
+  opts: SemanticSearchOpts,
+  topK: number,
+): Promise<RankedItem[][]> {
+  const rankedLists: RankedItem[][] = [];
+
+  for (const q of queries) {
+    const bm25Data = ftsSearchData(q, customDbPath, { ...opts, limit: topK });
+    if (bm25Data?.results) {
+      rankedLists.push(
+        bm25Data.results.map((r, idx) => ({
+          key: `${r.name}:${r.file}:${r.line}`,
+          rank: idx + 1,
+          source: 'bm25' as const,
+          ...r,
+        })),
+      );
+    }
+
+    const semData = await searchData(q, customDbPath, {
+      ...opts,
+      limit: topK,
+      minScore: opts.minScore ?? 0.2,
+    });
+    if (semData?.results) {
+      rankedLists.push(
+        semData.results.map((r, idx) => ({
+          key: `${r.name}:${r.file}:${r.line}`,
+          rank: idx + 1,
+          source: 'semantic' as const,
+          ...r,
+        })),
+      );
+    }
+  }
+
+  return rankedLists;
+}
+
+/** Initialise a fusion entry seeded from the first ranked item we see for a key. */
+function createFusionEntry(item: RankedItem): FusionEntry {
+  return {
+    name: item.name,
+    kind: item.kind,
+    file: item.file,
+    line: item.line,
+    endLine: (item.endLine as number | null) ?? null,
+    role: (item.role as string | null) ?? null,
+    fileHash: (item.fileHash as string | null) ?? null,
+    rrfScore: 0,
+    bm25Score: null,
+    bm25Rank: null,
+    similarity: null,
+    semanticRank: null,
+  };
+}
+
+/** Merge a single ranked item into its fusion entry: update RRF and best per-source rank. */
+function mergeRankedItem(entry: FusionEntry, item: RankedItem, k: number): void {
+  entry.rrfScore += 1 / (k + item.rank);
+  if (item.source === 'bm25') {
+    if (entry.bm25Rank === null || item.rank < entry.bm25Rank) {
+      entry.bm25Score = item.bm25Score ?? null;
+      entry.bm25Rank = item.rank;
+    }
+  } else if (entry.semanticRank === null || item.rank < entry.semanticRank) {
+    entry.similarity = item.similarity ?? null;
+    entry.semanticRank = item.rank;
+  }
+}
+
+/** Flatten a fusion entry into the public-facing hybrid result shape. */
+function toHybridResult(e: FusionEntry): HybridResult {
+  return {
+    name: e.name,
+    kind: e.kind,
+    file: e.file,
+    line: e.line,
+    endLine: e.endLine,
+    role: e.role,
+    fileHash: e.fileHash,
+    rrf: e.rrfScore,
+    bm25Score: e.bm25Score,
+    bm25Rank: e.bm25Rank,
+    similarity: e.similarity,
+    semanticRank: e.semanticRank,
+  };
+}
+
+/** Reciprocal Rank Fusion: merge ranked lists into a single scored result set. */
+function fuseResults(rankedLists: RankedItem[][], k: number, limit: number): HybridResult[] {
+  const fusionMap = new Map<string, FusionEntry>();
+  for (const list of rankedLists) {
+    for (const item of list) {
+      if (!fusionMap.has(item.key)) {
+        fusionMap.set(item.key, createFusionEntry(item));
+      }
+      mergeRankedItem(fusionMap.get(item.key)!, item, k);
+    }
+  }
+
+  return [...fusionMap.values()]
+    .sort((a, b) => b.rrfScore - a.rrfScore)
+    .slice(0, limit)
+    .map(toHybridResult);
+}
+
+export async function hybridSearchData(
+  query: string,
+  customDbPath: string | undefined,
+  opts: SemanticSearchOpts = {},
+): Promise<HybridSearchResult | null> {
+  const config = opts.config || loadConfig();
+  const searchCfg = config.search || ({} as CodegraphConfig['search']);
+  const limit = opts.limit ?? searchCfg.topK ?? 15;
+  const k = opts.rrfK ?? searchCfg.rrfK ?? 60;
+  const topK = (opts.limit ?? searchCfg.topK ?? 15) * 5;
+
+  const checkDb = openReadonlyOrFail(customDbPath) as BetterSqlite3Database;
+  const ftsAvailable = hasFtsIndex(checkDb);
+  checkDb.close();
+  if (!ftsAvailable) return null;
+
+  const queries = parseQueries(query);
+  const rankedLists = await collectRankedLists(queries, customDbPath, opts, topK);
+  const results = fuseResults(rankedLists, k, limit);
+
+  return { results };
+}

package/src/domain/search/search/keyword.ts

@@ -0,0 +1,101 @@
+import { openReadonlyOrFail } from '../../../db/index.js';
+import { buildFileConditionSQL } from '../../../db/query-builder.js';
+import type { BetterSqlite3Database } from '../../../types.js';
+import { normalizeSymbol } from '../../queries.js';
+import { hasFtsIndex, sanitizeFtsQuery } from '../stores/fts5.js';
+import { applyFilters } from './filters.js';
+
+export interface FtsSearchOpts {
+  limit?: number;
+  kind?: string;
+  filePattern?: string | string[];
+  noTests?: boolean;
+}
+
+interface FtsRow {
+  node_id: number;
+  bm25_score: number;
+  name: string;
+  kind: string;
+  file: string;
+  line: number;
+  end_line: number | null;
+  role: string | null;
+}
+
+export interface FtsSearchResult {
+  results: Array<{
+    name: string;
+    kind: string;
+    file: string;
+    line: number;
+    bm25Score: number;
+    [key: string]: unknown;
+  }>;
+}
+
+export function ftsSearchData(
+  query: string,
+  customDbPath: string | undefined,
+  opts: FtsSearchOpts = {},
+): FtsSearchResult | null {
+  const limit = opts.limit || 15;
+
+  const db = openReadonlyOrFail(customDbPath) as BetterSqlite3Database;
+
+  try {
+    if (!hasFtsIndex(db)) {
+      return null;
+    }
+
+    const ftsQuery = sanitizeFtsQuery(query);
+    if (!ftsQuery) {
+      return { results: [] };
+    }
+
+    let sql = `
+      SELECT f.rowid AS node_id, rank AS bm25_score,
+             n.name, n.kind, n.file, n.line, n.end_line, n.role
+      FROM fts_index f
+      JOIN nodes n ON f.rowid = n.id
+      WHERE fts_index MATCH ?
+    `;
+    const params: unknown[] = [ftsQuery];
+
+    if (opts.kind) {
+      sql += ' AND n.kind = ?';
+      params.push(opts.kind);
+    }
+
+    const fp = opts.filePattern;
+    const fpArr = Array.isArray(fp) ? fp : fp ? [fp] : [];
+    const isGlob = fpArr.length > 0 && fpArr.some((p) => /[*?[\]]/.test(p));
+    if (fpArr.length > 0 && !isGlob) {
+      const fc = buildFileConditionSQL(fpArr, 'n.file');
+      sql += fc.sql;
+      params.push(...fc.params);
+    }
+
+    sql += ' ORDER BY rank LIMIT ?';
+    params.push(limit * 5);
+
+    let rows: FtsRow[];
+    try {
+      rows = db.prepare(sql).all(...params) as FtsRow[];
+    } catch {
+      return { results: [] };
+    }
+
+    rows = applyFilters(rows, opts);
+
+    const hc = new Map<string, string>();
+    const results = rows.slice(0, limit).map((row) => ({
+      ...normalizeSymbol(row, db, hc),
+      bm25Score: -row.bm25_score,
+    }));
+
+    return { results };
+  } finally {
+    db.close();
+  }
+}

package/src/domain/search/search/prepare.ts

@@ -0,0 +1,95 @@
+import { openReadonlyOrFail } from '../../../db/index.js';
+import { escapeLike } from '../../../db/query-builder.js';
+import { getEmbeddingCount, getEmbeddingMeta } from '../../../db/repository/embeddings.js';
+import type { BetterSqlite3Database } from '../../../types.js';
+import { MODELS } from '../models.js';
+import { applyFilters } from './filters.js';
+
+export interface PreparedSearch {
+  db: BetterSqlite3Database;
+  rows: Array<{
+    node_id: number;
+    vector: Buffer;
+    text_preview: string;
+    name: string;
+    kind: string;
+    file: string;
+    line: number;
+    end_line: number | null;
+    role: string | null;
+  }>;
+  modelKey: string | null;
+  storedDim: number | null;
+}
+
+export interface PrepareSearchOpts {
+  model?: string;
+  kind?: string;
+  filePattern?: string | string[];
+  noTests?: boolean;
+}
+
+export function prepareSearch(
+  customDbPath: string | undefined,
+  opts: PrepareSearchOpts = {},
+): PreparedSearch | null {
+  const db = openReadonlyOrFail(customDbPath) as BetterSqlite3Database;
+
+  try {
+    const count = getEmbeddingCount(db);
+    if (count === 0) {
+      console.log('No embeddings found. Run `codegraph embed` first.');
+      db.close();
+      return null;
+    }
+
+    const storedModel = getEmbeddingMeta(db, 'model') || null;
+    const dimStr = getEmbeddingMeta(db, 'dim');
+    const storedDim = dimStr ? parseInt(dimStr, 10) : null;
+
+    let modelKey = opts.model || null;
+    if (!modelKey && storedModel) {
+      for (const [key, config] of Object.entries(MODELS)) {
+        if (config.name === storedModel) {
+          modelKey = key;
+          break;
+        }
+      }
+    }
+
+    const fp = opts.filePattern;
+    const fpArr = Array.isArray(fp) ? fp : fp ? [fp] : [];
+    const isGlob = fpArr.length > 0 && fpArr.some((p) => /[*?[\]]/.test(p));
+    let sql = `
+    SELECT e.node_id, e.vector, e.text_preview, n.name, n.kind, n.file, n.line, n.end_line, n.role
+    FROM embeddings e
+    JOIN nodes n ON e.node_id = n.id
+  `;
+    const params: unknown[] = [];
+    const conditions: string[] = [];
+    if (opts.kind) {
+      conditions.push('n.kind = ?');
+      params.push(opts.kind);
+    }
+    if (fpArr.length > 0 && !isGlob) {
+      if (fpArr.length === 1) {
+        conditions.push("n.file LIKE ? ESCAPE '\\'");
+        params.push(`%${escapeLike(fpArr[0]!)}%`);
+      } else {
+        conditions.push(`(${fpArr.map(() => "n.file LIKE ? ESCAPE '\\'").join(' OR ')})`);
+        params.push(...fpArr.map((f) => `%${escapeLike(f)}%`));
+      }
+    }
+    if (conditions.length > 0) {
+      sql += ` WHERE ${conditions.join(' AND ')}`;
+    }
+
+    let rows = db.prepare(sql).all(...params) as PreparedSearch['rows'];
+    rows = applyFilters(rows, opts);
+
+    return { db, rows, modelKey, storedDim };
+  } catch (err) {
+    db.close();
+    throw err;
+  }
+}