@monoes/monograph 1.5.6 → 1.5.7

package/src/storage/fts-store.ts

@@ -1,385 +0,0 @@
-import type Database from 'better-sqlite3';
-
-// ── Intent-to-identifier extraction ──────────────────────────────────────────
-
-const STOPWORDS = new Set([
-  'a','an','the','is','are','was','were','be','been','being','have','has','had',
-  'do','does','did','will','would','shall','should','may','might','must','can',
-  'could','am','it','its','i','me','my','we','our','you','your','he','she',
-  'they','them','their','this','that','these','those','of','in','to','for',
-  'with','on','at','from','by','about','as','into','through','during','before',
-  'after','above','below','between','out','off','over','under','again','then',
-  'once','here','there','when','where','why','how','all','both','each','every',
-  'few','more','most','other','some','such','no','nor','not','only','own',
-  'same','so','than','too','very','just','because','but','and','or','if',
-  'while','what','which','who','whom','up','down','get','set','add','use',
-  'find','show','make','let','want','need','like','look','also','new','old',
-]);
-
-/**
- * Extract code-relevant search terms from a natural-language query.
- * Strips stopwords, splits camelCase/PascalCase/snake_case, and strips
- * file extensions (so "CLAUDE.md" also tries "CLAUDE").
- * Returns an empty array when the query is already a bare identifier.
- */
-export function extractSearchTerms(query: string): string[] {
-  const words = query.split(/[\s,;:!?()\[\]{}'"]+/).filter(Boolean);
-  // If query is a single token, it's already a bare identifier query — nothing to extract.
-  // (We used to also bail out for short, stopword-free multi-token queries under the
-  // assumption they were "already an identifier query", but that misfires for queries
-  // like "ExtensionBridge keepalive reconnect" — three distinct identifier-ish tokens
-  // that FTS5 MATCH ANDs together and fails to find as a single row. Since this function
-  // only runs as a fallback after the raw MATCH already returned zero rows, there's no
-  // extra cost to always extracting terms for multi-token queries.)
-  if (words.length <= 1) return [];
-
-  const terms = new Set<string>();
-  for (const word of words) {
-    const lower = word.toLowerCase();
-    if (STOPWORDS.has(lower)) continue;
-    if (word.length < 3) continue;
-
-    // Strip file extensions: "CLAUDE.md" → also add "CLAUDE"
-    const dotIdx = word.lastIndexOf('.');
-    if (dotIdx > 0 && dotIdx < word.length - 1) {
-      terms.add(word.slice(0, dotIdx));
-      terms.add(word);
-    }
-
-    // Split camelCase/PascalCase: "ExtensionBridge" → "Extension", "Bridge"
-    const camelParts = word.replace(/([a-z])([A-Z])/g, '$1 $2')
-      .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
-      .split(/[\s_-]+/)
-      .filter(p => p.length >= 3 && !STOPWORDS.has(p.toLowerCase()));
-    if (camelParts.length > 1) {
-      for (const part of camelParts) terms.add(part);
-    }
-
-    // Keep the original word
-    terms.add(word);
-  }
-
-  return Array.from(terms);
-}
-
-export interface FtsResult {
-  id: string;
-  name: string;
-  normLabel: string;
-  filePath: string | null;
-  label: string;
-  rank: number;
-  /** First line of the symbol in its source file (1-based, null if unknown). */
-  startLine: number | null;
-  /** Last line of the symbol in its source file (1-based, null if unknown). */
-  endLine: number | null;
-}
-
-/**
- * Quote a single FTS5 search term as a string literal when it contains characters
- * that would otherwise be parsed as FTS5 syntax (quotes, parens, boolean keywords
- * like AND/OR/NOT, colons, hyphens, etc). Wrapping in double-quotes forces FTS5 to
- * treat the term as a literal string; any literal `"` inside the term is escaped by
- * doubling it, per FTS5's own string-literal quoting rules.
- */
-function quoteFtsTerm(term: string): string {
-  // Bare alphanumeric/underscore terms need no quoting and are the common case.
-  if (/^[A-Za-z0-9_]+$/.test(term)) return term;
-  const escaped = term.replace(/"/g, '""');
-  return `"${escaped}"`;
-}
-
-export function ftsSearch(
-  db: Database.Database,
-  query: string,
-  limit: number,
-  label?: string,
-): FtsResult[] {
-  // Sanitize: strip only characters that break FTS5 MATCH syntax (* still stripped,
-  // trigram handles substring natively; " is now preserved since trigram doesn't need it removed)
-  const safeQuery = query.replace(/[*]/g, ' ').trim();
-  if (!safeQuery) return [];
-
-  // Trigram handles substring matching natively — no need to append * to each term.
-  // Quote each term individually so characters like `"`, `(`, `)`, or bare boolean
-  // keywords (AND/OR/NOT) don't get parsed as FTS5 query syntax.
-  const ftsQuery = safeQuery
-    .split(/\s+/)
-    .map(quoteFtsTerm)
-    .join(' ');
-
-  let matchSql = `
-    SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
-           n.start_line, n.end_line, nodes_fts.rank
-    FROM nodes_fts
-    JOIN nodes n ON n.rowid = nodes_fts.rowid
-    WHERE nodes_fts MATCH ?
-  `;
-  const matchParams: unknown[] = [ftsQuery];
-  if (label) {
-    matchSql += ' AND n.label = ?';
-    matchParams.push(label);
-  }
-  matchSql += ' ORDER BY nodes_fts.rank LIMIT ?';
-  matchParams.push(limit);
-
-  let matchRows: Record<string, unknown>[] = [];
-  try {
-    matchRows = db.prepare(matchSql).all(...(matchParams as [string, ...unknown[]])) as Record<
-      string,
-      unknown
-    >[];
-  } catch {
-    // FTS MATCH can throw on malformed queries; fall through to LIKE path
-  }
-
-  // ── Intent-extraction fallback: when the raw query looks like natural language
-  // and FTS returned nothing, extract code-relevant tokens and retry FTS. ──────
-  if (matchRows.length === 0) {
-    const extracted = extractSearchTerms(safeQuery);
-    if (extracted.length > 0) {
-      // Try each extracted term individually via FTS, then merge by best rank
-      const termFtsQuery = extracted.map(quoteFtsTerm).join(' OR ');
-      let termSql = `
-        SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
-               n.start_line, n.end_line, nodes_fts.rank
-        FROM nodes_fts
-        JOIN nodes n ON n.rowid = nodes_fts.rowid
-        WHERE nodes_fts MATCH ?
-      `;
-      const termParams: unknown[] = [termFtsQuery];
-      if (label) {
-        termSql += ' AND n.label = ?';
-        termParams.push(label);
-      }
-      termSql += ' ORDER BY nodes_fts.rank LIMIT ?';
-      termParams.push(limit * 2);
-      try {
-        matchRows = db.prepare(termSql).all(...(termParams as [string, ...unknown[]])) as Record<string, unknown>[];
-      } catch { /* FTS MATCH can throw on malformed queries */ }
-
-      // Also try individual LIKE for each extracted term to catch camelCase substrings
-      if (matchRows.length < limit) {
-        const seenIds = new Set(matchRows.map((r) => r.id as string));
-        for (const term of extracted) {
-          if (matchRows.length >= limit) break;
-          const escapedTerm = term.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
-          const pat = `%${escapedTerm}%`;
-          let termLikeSql = `
-            SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
-                   n.start_line, n.end_line, 0 AS rank
-            FROM nodes n
-            WHERE (n.name LIKE ? ESCAPE '\\' OR n.norm_label LIKE ? ESCAPE '\\')
-          `;
-          const termLikeParams: unknown[] = [pat, pat];
-          if (label) {
-            termLikeSql += ' AND n.label = ?';
-            termLikeParams.push(label);
-          }
-          termLikeSql += ' LIMIT ?';
-          termLikeParams.push(limit);
-          const likeRows = db.prepare(termLikeSql).all(...(termLikeParams as [string, ...unknown[]])) as Record<string, unknown>[];
-          for (const r of likeRows) {
-            if (!seenIds.has(r.id as string)) {
-              matchRows.push(r);
-              seenIds.add(r.id as string);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  // Run the LIKE fallback whenever MATCH threw (malformed/boolean-keyword query) or
-  // returned zero rows — not just for short (≤2 char) queries. Short queries need it
-  // because trigram requires ≥3 characters to fire; longer queries need it whenever
-  // MATCH couldn't handle the syntax, so a query-syntax failure doesn't silently read
-  // as "no results".
-  if (safeQuery.length <= 2 || matchRows.length === 0) {
-    const escapedQuery = safeQuery.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
-    const likePattern = `%${escapedQuery}%`;
-
-    // Also try without file extension: "CLAUDE.md" → "CLAUDE"
-    const dotIdx = safeQuery.lastIndexOf('.');
-    const strippedLikePattern = (dotIdx > 0 && dotIdx < safeQuery.length - 1)
-      ? `%${safeQuery.slice(0, dotIdx).replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_')}%`
-      : null;
-
-    let likeSql = `
-      SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
-             n.start_line, n.end_line, 0 AS rank
-      FROM nodes n
-      WHERE (n.name LIKE ? ESCAPE '\\' OR n.norm_label LIKE ? ESCAPE '\\' OR n.file_path LIKE ? ESCAPE '\\'`;
-    const likeParams: unknown[] = [likePattern, likePattern, likePattern];
-    if (strippedLikePattern) {
-      likeSql += ` OR n.name LIKE ? ESCAPE '\\'`;
-      likeParams.push(strippedLikePattern);
-    }
-    likeSql += ')';
-    if (label) {
-      likeSql += ' AND n.label = ?';
-      likeParams.push(label);
-    }
-    likeSql += ' LIMIT ?';
-    likeParams.push(limit);
-
-    const likeRows = db.prepare(likeSql).all(...(likeParams as [string, ...unknown[]])) as Record<
-      string,
-      unknown
-    >[];
-
-    // Merge: MATCH results first, append LIKE results not already present
-    const seenIds = new Set(matchRows.map((r) => r.id as string));
-    for (const r of likeRows) {
-      if (!seenIds.has(r.id as string)) {
-        matchRows.push(r);
-        seenIds.add(r.id as string);
-      }
-    }
-  }
-
-  return matchRows.slice(0, limit).map((r) => ({
-    id: r.id as string,
-    name: r.name as string,
-    normLabel: r.norm_label as string,
-    filePath: (r.file_path as string | null) ?? null,
-    label: r.label as string,
-    rank: r.rank as number,
-    startLine: (r.start_line as number | null) ?? null,
-    endLine: (r.end_line as number | null) ?? null,
-  }));
-}
-
-// ── Hybrid search ─────────────────────────────────────────────────────────────
-
-export interface HybridSearchResult extends FtsResult {
-  combinedScore: number;
-  matchStrategy: 'fts' | 'like' | 'fuzzy';
-}
-
-/**
- * Computes an in-memory fuzzy sequence score for `query` against `name`.
- * Walks through query chars left-to-right, finding each in `name` sequentially.
- * Returns a score in [0, 1]: higher means a tighter match.
- */
-function computeFuzzyScore(name: string, query: string): number {
-  if (!query.length) return 0;
-  const lname = name.toLowerCase();
-  const lquery = query.toLowerCase();
-  let nameIdx = 0;
-  let matched = 0;
-  let gapPenalty = 0;
-  let lastMatchPos = -1;
-
-  for (let qi = 0; qi < lquery.length; qi++) {
-    const ch = lquery[qi];
-    const pos = lname.indexOf(ch, nameIdx);
-    if (pos === -1) break;
-    matched++;
-    if (lastMatchPos !== -1) {
-      gapPenalty += pos - lastMatchPos - 1;
-    }
-    lastMatchPos = pos;
-    nameIdx = pos + 1;
-  }
-
-  if (matched === 0) return 0;
-  const matchRatio = matched / lquery.length;
-  const normalizedGap = gapPenalty / (lname.length || 1);
-  return matchRatio * (1 / (1 + normalizedGap));
-}
-
-/** Returns a small bonus based on node label to favour structural node types. */
-function computeNodeTypeBonus(label: string): number {
-  if (label === 'File' || label === 'Module') return 0.02;
-  if (label === 'Class') return 0.01;
-  return 0;
-}
-
-/**
- * Hybrid search combining three strategies:
- *  1. FTS5 (trigram) BM25 match via `ftsSearch`
- *  2. LIKE fallback for short queries (≤3 chars) or when FTS returns 0 results
- *  3. In-memory fuzzy character-sequence scoring applied to all candidates
- *
- * Results are deduped by id (highest combinedScore wins), re-ranked, and
- * trimmed to `limit`. The existing `ftsSearch` is left unchanged.
- */
-export function hybridSearch(
-  db: Database.Database,
-  query: string,
-  limit: number,
-  label?: string,
-): HybridSearchResult[] {
-  const safeQuery = query.replace(/[*]/g, ' ').trim();
-  if (!safeQuery) return [];
-
-  // id → best result so far
-  const best = new Map<string, HybridSearchResult>();
-
-  const upsert = (result: HybridSearchResult): void => {
-    const existing = best.get(result.id);
-    if (!existing || result.combinedScore > existing.combinedScore) {
-      best.set(result.id, result);
-    }
-  };
-
-  // ── Strategy 1: FTS5 BM25 ──────────────────────────────────────────────────
-  const ftsRows = ftsSearch(db, safeQuery, limit * 2, label);
-  for (const row of ftsRows) {
-    // FTS5 rank is negative; larger magnitude = better match. Map to (0,1) with higher = better.
-    const absRank = Math.abs(row.rank);
-    const ftsScore = absRank / (1 + absRank);
-    const fuzz = computeFuzzyScore(row.name, safeQuery);
-    const combined = ftsScore + fuzz + computeNodeTypeBonus(row.label);
-    upsert({ ...row, combinedScore: combined, matchStrategy: 'fts' });
-  }
-
-  // ── Strategy 2: LIKE fallback ──────────────────────────────────────────────
-  // Always run for short queries (≤3 chars) or when FTS returned nothing.
-  if (safeQuery.length <= 3 || ftsRows.length === 0) {
-    const escapedSafeQuery = safeQuery.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
-    const likePattern = `%${escapedSafeQuery}%`;
-    let likeSql = `
-      SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
-             n.start_line, n.end_line
-      FROM nodes n
-      WHERE (n.name LIKE ? ESCAPE '\\' OR n.norm_label LIKE ? ESCAPE '\\')
-    `;
-    const likeParams: unknown[] = [likePattern, likePattern];
-    if (label) {
-      likeSql += ' AND n.label = ?';
-      likeParams.push(label);
-    }
-    likeSql += ' LIMIT ?';
-    likeParams.push(limit * 2);
-
-    const likeRows = db
-      .prepare(likeSql)
-      .all(...(likeParams as [string, ...unknown[]])) as Record<string, unknown>[];
-
-    for (const r of likeRows) {
-      const name = r.name as string;
-      const lbl = r.label as string;
-      const fuzz = computeFuzzyScore(name, safeQuery);
-      const combined = 0.3 + fuzz + computeNodeTypeBonus(lbl);
-      upsert({
-        id: r.id as string,
-        name,
-        normLabel: r.norm_label as string,
-        filePath: (r.file_path as string | null) ?? null,
-        label: lbl,
-        rank: 0,
-        startLine: (r.start_line as number | null) ?? null,
-        endLine: (r.end_line as number | null) ?? null,
-        combinedScore: combined,
-        matchStrategy: 'like',
-      });
-    }
-  }
-
-  // ── Sort, dedupe (handled by Map), slice ───────────────────────────────────
-  return Array.from(best.values())
-    .sort((a, b) => b.combinedScore - a.combinedScore)
-    .slice(0, limit);
-}

package/src/storage/node-store.ts

@@ -1,209 +0,0 @@
-import type Database from 'better-sqlite3';
-import type { MonographNode } from '../types.js';
-import { toNormLabel, MonographError } from '../types.js';
-
-export function insertNode(db: Database.Database, node: MonographNode): void {
-  db.prepare(`
-    INSERT OR REPLACE INTO nodes
-      (id, label, name, norm_label, file_path, start_line, end_line,
-       community_id, is_exported, language, properties)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-  `).run(
-    node.id,
-    node.label,
-    node.name,
-    node.normLabel ?? toNormLabel(node.name),
-    node.filePath ?? null,
-    node.startLine ?? null,
-    node.endLine ?? null,
-    node.communityId ?? null,
-    node.isExported ? 1 : 0,
-    node.language ?? null,
-    node.properties ? JSON.stringify(node.properties) : null,
-  );
-}
-
-const INSERT_SQL = `
-    INSERT OR REPLACE INTO nodes
-      (id, label, name, norm_label, file_path, start_line, end_line,
-       community_id, is_exported, language, properties)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-  `;
-
-export function insertNodes(db: Database.Database, nodes: MonographNode[]): void {
-  if (nodes.length === 0) return;
-  // Prepare once and reuse across all rows — avoids N redundant prepare calls
-  const stmt = db.prepare(INSERT_SQL);
-  const insertMany = db.transaction((rows: MonographNode[]) => {
-    for (const n of rows) {
-      stmt.run(
-        n.id,
-        n.label,
-        n.name,
-        n.normLabel ?? toNormLabel(n.name),
-        n.filePath ?? null,
-        n.startLine ?? null,
-        n.endLine ?? null,
-        n.communityId ?? null,
-        n.isExported ? 1 : 0,
-        n.language ?? null,
-        n.properties ? JSON.stringify(n.properties) : null,
-      );
-    }
-  });
-  insertMany(nodes);
-}
-
-/** Batch-fetch multiple nodes by ID using a single SQL IN query. Batches at 50 IDs. */
-export function getNodesByIds(db: Database.Database, ids: string[]): MonographNode[] {
-  if (ids.length === 0) return [];
-  const BATCH = 50;
-  const results: MonographNode[] = [];
-  for (let i = 0; i < ids.length; i += BATCH) {
-    const chunk = ids.slice(i, i + BATCH);
-    const placeholders = chunk.map(() => '?').join(',');
-    const rows = db
-      .prepare(`SELECT * FROM nodes WHERE id IN (${placeholders})`)
-      .all(...chunk) as Record<string, unknown>[];
-    for (const row of rows) results.push(rowToNode(row));
-  }
-  return results;
-}
-
-export function getNode(db: Database.Database, id: string): MonographNode | undefined {
-  const row = db
-    .prepare('SELECT * FROM nodes WHERE id = ?')
-    .get(id) as Record<string, unknown> | undefined;
-  return row ? rowToNode(row) : undefined;
-}
-
-export function getNodesForFile(db: Database.Database, filePath: string): MonographNode[] {
-  const rows = db
-    .prepare('SELECT * FROM nodes WHERE file_path = ?')
-    .all(filePath) as Record<string, unknown>[];
-  return rows.map(rowToNode);
-}
-
-export function deleteNodesForFile(db: Database.Database, filePath: string): void {
-  db.prepare('DELETE FROM nodes WHERE file_path = ?').run(filePath);
-}
-
-export function countNodes(db: Database.Database): number {
-  const row = db.prepare('SELECT COUNT(*) as n FROM nodes').get() as { n: number };
-  return row.n;
-}
-
-// ── Property registry ─────────────────────────────────────────────────────────
-
-export interface PropertyDef {
-  ident: string;
-  type: string;
-  cardinality: string;
-  viewContext: string;
-  closedValues: string[] | null;
-  description: string | null;
-  queryable: boolean;
-}
-
-function rowToPropDef(row: Record<string, unknown>): PropertyDef {
-  return {
-    ident: row.ident as string,
-    type: row.type as string,
-    cardinality: row.cardinality as string,
-    viewContext: row.view_context as string,
-    closedValues: row.closed_values ? JSON.parse(row.closed_values as string) : null,
-    description: (row.description as string | null) ?? null,
-    queryable: (row.queryable as number) === 1,
-  };
-}
-
-/** List all registered property definitions */
-export function listProperties(db: Database.Database): PropertyDef[] {
-  const rows = db.prepare('SELECT * FROM node_properties ORDER BY ident').all() as Record<string, unknown>[];
-  return rows.map(rowToPropDef);
-}
-
-/** Get a single property definition */
-export function getProperty(db: Database.Database, ident: string): PropertyDef | null {
-  const row = db.prepare('SELECT * FROM node_properties WHERE ident = ?').get(ident) as Record<string, unknown> | undefined;
-  return row ? rowToPropDef(row) : null;
-}
-
-/** Register or update a custom property */
-export function upsertProperty(db: Database.Database, def: PropertyDef): void {
-  db.prepare(`
-    INSERT INTO node_properties (ident, type, cardinality, view_context, closed_values, description, queryable)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(ident) DO UPDATE SET
-      type = excluded.type,
-      cardinality = excluded.cardinality,
-      view_context = excluded.view_context,
-      closed_values = excluded.closed_values,
-      description = excluded.description,
-      queryable = excluded.queryable
-  `).run(
-    def.ident,
-    def.type,
-    def.cardinality,
-    def.viewContext,
-    def.closedValues != null ? JSON.stringify(def.closedValues) : null,
-    def.description ?? null,
-    def.queryable ? 1 : 0,
-  );
-}
-
-/**
- * Query nodes by a typed property value extracted from their JSON properties column.
- * For queryable properties only.
- * @param ident - property ident (e.g. 'layer', 'tags', 'ua_type')
- * @param value - value to match (exact for closed/text, numeric comparison for number)
- * @param comparator - '=' | 'LIKE' | '>' | '<' (default '=')
- */
-export function queryByProperty(
-  db: Database.Database,
-  ident: string,
-  value: string | number | boolean,
-  comparator: '=' | 'LIKE' | '>' | '<' = '=',
-  limit = 100,
-): Array<{ id: string; name: string; label: string; filePath: string | null; propertyValue: unknown }> {
-  const propDef = getProperty(db, ident);
-  if (!propDef) {
-    throw new MonographError(`Unknown property: '${ident}'. Register it first with upsertProperty.`);
-  }
-  if (!propDef.queryable) {
-    throw new MonographError(`Property '${ident}' is not queryable (view_context may be 'never' or queryable=false).`);
-  }
-
-  const extractExpr = `json_extract(properties, '$.${ident}')`;
-  const rows = db.prepare(`
-    SELECT id, name, label, file_path, ${extractExpr} AS property_value
-    FROM nodes
-    WHERE properties IS NOT NULL
-      AND ${extractExpr} ${comparator} ?
-    LIMIT ?
-  `).all(value, limit) as Array<Record<string, unknown>>;
-
-  return rows.map(r => ({
-    id: r.id as string,
-    name: r.name as string,
-    label: r.label as string,
-    filePath: (r.file_path as string | null) ?? null,
-    propertyValue: r.property_value,
-  }));
-}
-
-function rowToNode(row: Record<string, unknown>): MonographNode {
-  return {
-    id: row.id as string,
-    label: row.label as MonographNode['label'],
-    name: row.name as string,
-    normLabel: row.norm_label as string,
-    filePath: row.file_path as string | undefined,
-    startLine: row.start_line as number | undefined,
-    endLine: row.end_line as number | undefined,
-    communityId: row.community_id as number | undefined,
-    isExported: (row.is_exported as number) === 1,
-    language: row.language as string | undefined,
-    properties: row.properties ? JSON.parse(row.properties as string) : undefined,
-  };
-}