@monoes/monograph 1.5.6 → 1.5.8

package/src/pipeline/phases/parse.ts

@@ -1,222 +0,0 @@
-import { readFileSync, statSync } from 'fs';
-import { extname, join } from 'path';
-import type { PipelinePhase, PipelineContext } from '../types.js';
-import type { MonographNode, MonographEdge } from '../../types.js';
-import { parseFile } from '../../parsers/loader.js';
-import { insertNodes, deleteNodesForFile } from '../../storage/node-store.js';
-import { insertEdges, deleteEdgesForFile } from '../../storage/edge-store.js';
-import type { StructureOutput } from './structure.js';
-import { extractVariables, variableToNode } from './variables.js';
-import { ExtractionCache } from '../../cache/extraction-cache.js';
-
-export interface ParseOutput {
-  symbolNodes: MonographNode[];
-  allEdges: MonographEdge[];
-  parseErrors: string[];
-  fileContents: Map<string, string>;
-  cacheStats: { hits: number; misses: number };
-}
-
-export const parsePhase: PipelinePhase<ParseOutput> = {
-  name: 'parse',
-  deps: ['structure'],
-  async execute(ctx, deps) {
-    const { fileNodes } = deps.get('structure') as StructureOutput;
-    const symbolNodes: MonographNode[] = [];
-    const allEdges: MonographEdge[] = [];
-    // freshNodes/freshEdges (the cache-miss-only subset) are NOT accumulated in a
-    // parallel array during the loop — that duplicated every fresh node/edge object
-    // in memory for the entire pipeline run. Instead we record the [start, end)
-    // slice of symbolNodes/allEdges that each cache-miss file contributed, and
-    // materialize the fresh subset via a single slice pass at the end, only when
-    // it's actually needed (cacheHits > 0). On a cold build (cacheHits === 0) the
-    // fresh subset is never materialized at all, since nodesToInsert falls back to
-    // symbolNodes/allEdges directly in that case — eliminating the redundant
-    // allocation entirely for the memory-heaviest scenario.
-    const freshNodeRanges: Array<[number, number]> = [];
-    const freshEdgeRanges: Array<[number, number]> = [];
-    const staleFilePaths: string[] = [];
-    const parseErrors: string[] = [];
-    const fileContents = new Map<string, string>();
-    let processed = 0;
-    let cacheHits = 0;
-    let cacheMisses = 0;
-
-    const cache = new ExtractionCache(join(ctx.repoPath, '.monomind', 'parse-cache'));
-
-    for (const fileNode of fileNodes) {
-      const absPath = fileNode.filePath ? `${ctx.repoPath}/${fileNode.filePath}` : '';
-      const ext = extname(absPath).toLowerCase();
-      if (ext === '.md' || ext === '.markdown') { processed++; continue; }
-
-      // Fast path: mtime+size check avoids reading file content entirely.
-      // Skipped entirely under --force so a force rebuild is a genuine from-scratch
-      // parse, not a replay of whatever extraction the cache happens to hold.
-      const cached = ctx.options.force ? null : cache.getWithStat(absPath);
-      if (cached) {
-        symbolNodes.push(...cached.nodes);
-        allEdges.push(...cached.edges);
-        cacheHits++;
-      } else {
-        let source: string;
-        try {
-          const stat = statSync(absPath);
-          if (stat.size > ctx.options.maxFileSizeBytes) {
-            parseErrors.push(`${fileNode.filePath}: skipped (too large)`);
-            continue;
-          }
-          source = readFileSync(absPath, 'utf-8');
-        } catch (err) {
-          const code = (err as NodeJS.ErrnoException)?.code;
-          const detail = code || (err instanceof Error ? err.message : String(err));
-          parseErrors.push(`${fileNode.filePath}: unreadable (${detail})`);
-          continue;
-        }
-        fileContents.set(fileNode.filePath ?? absPath, source);
-        const result = await parseFile(absPath, source, fileNode.filePath ?? '');
-        const fileSymbols: MonographNode[] = [...result.nodes];
-        const fileEdges: MonographEdge[] = [...result.edges];
-        parseErrors.push(...result.parseErrors);
-
-        if (ext === '.cs') {
-          const csNamespaces = extractCsharpNamespaces(source, fileNode.filePath ?? '');
-          for (const ns of csNamespaces) {
-            fileSymbols.push({
-              id: `${ns.filePath}::namespace::${ns.name}`,
-              name: ns.name,
-              label: 'Namespace',
-              normLabel: 'namespace',
-              filePath: ns.filePath,
-              line: ns.line,
-              isExported: true,
-            } as import('../../types.js').MonographNode);
-          }
-        }
-
-        if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) {
-          const varInfos = extractVariables(source, fileNode.filePath ?? '');
-          fileSymbols.push(...varInfos.map(v => variableToNode(v)));
-
-          const arrowFns = extractArrowFunctions(source, fileNode.filePath ?? '');
-          for (const fn of arrowFns) {
-            fileSymbols.push({
-              id: `${fn.filePath}::fn::${fn.name}`,
-              name: fn.name,
-              label: 'Function',
-              normLabel: 'function',
-              filePath: fn.filePath,
-              line: fn.line,
-              isExported: fn.isExported,
-            } as import('../../types.js').MonographNode);
-          }
-        }
-
-        try {
-          const fileHash = cache.hashContent(source);
-          cache.setDeferred(absPath, fileHash, fileSymbols, fileEdges);
-        } catch { /* non-fatal */ }
-        const nodeStart = symbolNodes.length;
-        const edgeStart = allEdges.length;
-        symbolNodes.push(...fileSymbols);
-        allEdges.push(...fileEdges);
-        freshNodeRanges.push([nodeStart, symbolNodes.length]);
-        freshEdgeRanges.push([edgeStart, allEdges.length]);
-        // This file was re-parsed (cache miss) — its OLD node/edge set (from the
-        // previous build) must be deleted before the fresh rows are inserted below,
-        // otherwise a renamed/removed symbol's old row survives forever (ghost rows).
-        if (fileNode.filePath) staleFilePaths.push(fileNode.filePath);
-        cacheMisses++;
-      }
-      processed++;
-
-      if (processed % 50 === 0) {
-        ctx.onProgress?.({ phase: 'parse', filesProcessed: processed, totalFiles: fileNodes.length });
-      }
-    }
-
-    cache.flush();
-
-    if (ctx.db) {
-      const db = ctx.db;
-      // Only insert freshly-parsed nodes — cached nodes are already in the DB
-      // from the previous build (node IDs are deterministic from file_path + symbol).
-      // On first build (no cache hits), freshNodes === symbolNodes, so the fresh
-      // subset is never materialized — symbolNodes/allEdges are used directly.
-      const freshNodes = cacheHits > 0
-        ? freshNodeRanges.flatMap(([s, e]) => symbolNodes.slice(s, e))
-        : symbolNodes;
-      const freshEdges = cacheHits > 0
-        ? freshEdgeRanges.flatMap(([s, e]) => allEdges.slice(s, e))
-        : allEdges;
-      const nodesToInsert = freshNodes;
-      const knownIds = new Set(symbolNodes.map(n => n.id));
-      const edgesToInsert = freshEdges.filter(e => knownIds.has(e.targetId));
-
-      // For every cache-miss file, purge its OLD node/edge set BEFORE inserting the
-      // fresh parse results, inside the same transaction — otherwise a renamed or
-      // removed symbol's old row would survive forever (INSERT OR REPLACE only
-      // overwrites rows whose id still matches; it never deletes rows whose id
-      // disappeared because the symbol was renamed).
-      const writeAll = db.transaction(() => {
-        for (const filePath of staleFilePaths) {
-          deleteEdgesForFile(db, filePath);
-          deleteNodesForFile(db, filePath);
-        }
-        insertNodes(db, nodesToInsert);
-        insertEdges(db, edgesToInsert);
-      });
-      writeAll();
-    }
-
-    if (cacheHits > 0 && cacheMisses === 0) {
-      ctx.allFilesCached = true;
-    }
-
-    if (cacheHits > 0) {
-      const freshNodeCount = freshNodeRanges.reduce((sum, [s, e]) => sum + (e - s), 0);
-      ctx.onProgress?.({ phase: 'parse', filesProcessed: processed, totalFiles: fileNodes.length,
-        message: `cache: ${cacheHits} hits, ${cacheMisses} misses (${freshNodeCount} nodes inserted)` } as any);
-    }
-
-    return { symbolNodes, allEdges, parseErrors, fileContents, cacheStats: { hits: cacheHits, misses: cacheMisses } };
-  },
-};
-
-export function extractCsharpNamespaces(
-  source: string,
-  filePath: string,
-): Array<{ name: string; label: 'Namespace'; filePath: string; line: number }> {
-  const results: Array<{ name: string; label: 'Namespace'; filePath: string; line: number }> = [];
-  const re = /^[ \t]*namespace\s+([\w.]+)\s*[\{;]/gm;
-  let m: RegExpExecArray | null;
-  while ((m = re.exec(source)) !== null) {
-    const charsBefore = source.slice(0, m.index);
-    const line = (charsBefore.match(/\n/g)?.length ?? 0) + 1;
-    results.push({ name: m[1]!, label: 'Namespace', filePath, line });
-  }
-  return results;
-}
-
-export function extractArrowFunctions(
-  source: string,
-  filePath: string,
-): Array<{ name: string; isExported: boolean; line: number; filePath: string }> {
-  const results: Array<{ name: string; isExported: boolean; line: number; filePath: string }> = [];
-  // Match: (export)? const/let NAME = (async)? (...) =>
-  const re = /^([ \t]*)(export\s+)?(?:const|let)\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>/gm;
-
-  let m: RegExpExecArray | null;
-  re.lastIndex = 0;
-  while ((m = re.exec(source)) !== null) {
-    const charsBefore = source.slice(0, m.index);
-    const lineNum = (charsBefore.match(/\n/g)?.length ?? 0) + 1;
-    results.push({
-      name: m[3]!,
-      isExported: !!(m[2]?.trim()),
-      line: lineNum,
-      filePath,
-    });
-  }
-
-  return results;
-}

package/src/pipeline/phases/pdf-parse.ts

@@ -1,133 +0,0 @@
-import { readFileSync, statSync, readdirSync } from 'fs';
-import { extname, basename, join, relative } from 'path';
-import type { PipelinePhase, PipelineContext } from '../types.js';
-import type { MonographNode, MonographEdge } from '../../types.js';
-import { makeId, toNormLabel, CONFIDENCE_SCORE } from '../../types.js';
-import { insertNodes } from '../../storage/node-store.js';
-import { insertEdges } from '../../storage/edge-store.js';
-
-const IGNORE_DIRS = new Set([
-  'node_modules', '.git', 'dist', 'build', '__pycache__',
-  '.cache', 'coverage', '.monomind', 'vendor', 'target',
-  '.worktrees',
-]);
-
-export interface PdfParseOutput {
-  sectionNodes: MonographNode[];
-  pdfFiles: number;
-}
-
-function walkPdfs(dir: string, repoPath: string, ignore: string[]): string[] {
-  const extraIgnore = new Set(ignore);
-  const results: string[] = [];
-  function walk(d: string) {
-    let entries: string[];
-    try { entries = readdirSync(d); } catch { return; }
-    for (const entry of entries) {
-      if (IGNORE_DIRS.has(entry) || extraIgnore.has(entry)) continue;
-      const full = join(d, entry);
-      let st;
-      try { st = statSync(full); } catch { continue; }
-      if (st.isDirectory()) { walk(full); continue; }
-      if (extname(entry).toLowerCase() === '.pdf') results.push(full);
-    }
-  }
-  walk(dir);
-  return results;
-}
-
-export const pdfParsePhase: PipelinePhase<PdfParseOutput> = {
-  name: 'pdf-parse',
-  deps: ['structure'],
-
-  async execute(ctx: PipelineContext) {
-    if (ctx.options.codeOnly) return { sectionNodes: [], pdfFiles: 0 };
-
-    // Dynamically load pdf-parse — skip silently if not installed.
-    // Use Function() to prevent TypeScript from resolving the optional module at build time.
-    let extractText: ((buf: Buffer) => Promise<string>) | null = null;
-    try {
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const mod = await (new Function('return import("pdf-parse")')() as Promise<any>);
-      const fn = mod.default ?? mod;
-      extractText = async (buf: Buffer) => (await fn(buf)).text as string;
-    } catch {
-      return { sectionNodes: [], pdfFiles: 0 };
-    }
-
-    const pdfPaths = walkPdfs(ctx.repoPath, ctx.repoPath, ctx.options.ignore);
-    if (pdfPaths.length === 0) return { sectionNodes: [], pdfFiles: 0 };
-
-    const sectionNodes: MonographNode[] = [];
-    const allEdges: MonographEdge[] = [];
-
-    for (const absPath of pdfPaths) {
-      const rel = relative(ctx.repoPath, absPath);
-      const fileId = makeId(rel.replace(/\//g, '_'), 'file');
-
-      let text: string;
-      try {
-        const buf = readFileSync(absPath);
-        if (buf.length > ctx.options.maxFileSizeBytes * 10) continue; // 5 MB limit for PDFs
-        text = await extractText(buf);
-      } catch { continue; }
-
-      if (!text || text.trim().length === 0) continue;
-
-      // Split into ~1500-char paragraphs with 150-char overlap (knowledge_graph style)
-      const CHUNK = 1500;
-      const OVERLAP = 150;
-      const chunks: Array<{ content: string; startChar: number }> = [];
-      for (let pos = 0; pos < text.length; pos += CHUNK - OVERLAP) {
-        chunks.push({ content: text.slice(pos, pos + CHUNK), startChar: pos });
-        if (pos + CHUNK >= text.length) break;
-      }
-
-      const docTitle = basename(rel).replace(/\.pdf$/i, '');
-
-      for (let i = 0; i < chunks.length; i++) {
-        const { content, startChar } = chunks[i];
-        const title = `${docTitle} §${i + 1}`;
-        const sectionId = makeId('section', fileId, String(i));
-
-        const node: MonographNode = {
-          id: sectionId, label: 'Section',
-          name: title, normLabel: toNormLabel(title),
-          filePath: rel, startLine: startChar, endLine: startChar + content.length,
-          isExported: false,
-          properties: { level: 1, content: content.slice(0, 2000), source: 'pdf', chunk: i },
-        };
-        sectionNodes.push(node);
-
-        allEdges.push({
-          id: makeId(fileId, sectionId, 'defines'),
-          sourceId: fileId, targetId: sectionId,
-          relation: 'DEFINES', confidence: 'EXTRACTED', confidenceScore: CONFIDENCE_SCORE.EXTRACTED,
-        });
-      }
-    }
-
-    if (sectionNodes.length > 0) {
-      // File nodes are not persisted by structure/parse phases for non-code files —
-      // insert them here before edges to satisfy the FK constraint.
-      const seenFileIds = new Set<string>();
-      const fileNodes: MonographNode[] = [];
-      for (const edge of allEdges) {
-        if (!seenFileIds.has(edge.sourceId)) {
-          seenFileIds.add(edge.sourceId);
-          const relPath = sectionNodes.find(n => n.id === edge.targetId)?.filePath ?? '';
-          fileNodes.push({
-            id: edge.sourceId, label: 'File',
-            name: basename(relPath), normLabel: toNormLabel(basename(relPath)),
-            filePath: relPath, isExported: false,
-          });
-        }
-      }
-      insertNodes(ctx.db, [...fileNodes, ...sectionNodes]);
-      insertEdges(ctx.db, allEdges);
-    }
-
-    ctx.onProgress?.({ phase: 'pdf-parse', message: `Parsed ${pdfPaths.length} PDF files → ${sectionNodes.length} chunks` });
-    return { sectionNodes, pdfFiles: pdfPaths.length };
-  },
-};

package/src/pipeline/phases/processes.ts

@@ -1,249 +0,0 @@
-import { basename, extname } from 'path';
-import type { PipelinePhase } from '../types.js';
-import type { MonographNode, MonographEdge } from '../../types.js';
-import { makeId, toNormLabel } from '../../types.js';
-import { insertNodes } from '../../storage/node-store.js';
-import { insertEdges } from '../../storage/edge-store.js';
-import type Database from 'better-sqlite3';
-
-// ── Output types ──────────────────────────────────────────────────────────────
-
-export interface ProcessDef {
-  id: string;
-  name: string;
-  filePath: string;
-  entryNodeId: string;
-  stepCount: number;
-}
-
-export interface ProcessesOutput {
-  processResult: {
-    processes: ProcessDef[];
-    memberships: Map<string, string>; // nodeId → processId
-    stats: { totalProcesses: number; totalSteps: number };
-  };
-}
-
-// ── Entry-point file names that boost score ───────────────────────────────────
-
-const ENTRY_FILENAMES = new Set(['index', 'main', 'cli', 'server', 'app', 'cmd']);
-
-// ── BFS call-chain traversal ──────────────────────────────────────────────────
-
-function bfsCallChain(startNodeId: string, db: Database.Database, maxDepth = 8): Set<string> {
-  const visited = new Set<string>([startNodeId]);
-  const queue: Array<{ id: string; depth: number }> = [{ id: startNodeId, depth: 0 }];
-
-  const calleesStmt = db.prepare(`
-    SELECT target_id FROM edges WHERE source_id = ? AND relation = 'CALLS'
-  `);
-
-  while (queue.length > 0) {
-    const item = queue.shift()!;
-    if (item.depth >= maxDepth) continue;
-
-    const callees = calleesStmt.all(item.id) as Array<{ target_id: string }>;
-    for (const { target_id } of callees) {
-      if (!visited.has(target_id)) {
-        visited.add(target_id);
-        queue.push({ id: target_id, depth: item.depth + 1 });
-      }
-    }
-  }
-
-  return visited;
-}
-
-// ── Entry-point scoring ───────────────────────────────────────────────────────
-
-interface ScoredNode {
-  id: string;
-  name: string;
-  filePath: string;
-  language: string;
-  score: number;
-}
-
-function scoreEntryPoints(db: Database.Database): ScoredNode[] {
-  const symbolRows = db
-    .prepare(
-      `SELECT id, name, file_path, is_exported, language
-       FROM nodes
-       WHERE label IN ('Function', 'Method', 'Class')`,
-    )
-    .all() as Array<{
-      id: string;
-      name: string;
-      file_path: string | null;
-      is_exported: number;
-      language: string | null;
-    }>;
-
-  // Preload all three edge-count signals in a single pass instead of 3 per-row queries.
-  // LEFT JOIN with FILTER avoids N*3 prepared-stmt round-trips.
-  const edgeCounts = db
-    .prepare(
-      `SELECT
-         n.id,
-         COUNT(CASE WHEN e.relation = 'CALLS'         THEN 1 END) AS incoming_calls,
-         COUNT(CASE WHEN e.relation = 'HANDLES_ROUTE' THEN 1 END) AS route_count,
-         COUNT(CASE WHEN e.relation = 'HANDLES_TOOL'  THEN 1 END) AS tool_count
-       FROM nodes n
-       LEFT JOIN edges e ON e.target_id = n.id AND e.relation IN ('CALLS','HANDLES_ROUTE','HANDLES_TOOL')
-       WHERE n.label IN ('Function','Method','Class')
-       GROUP BY n.id`,
-    )
-    .all() as Array<{ id: string; incoming_calls: number; route_count: number; tool_count: number }>;
-
-  const callsMap    = new Map<string, number>();
-  const routeSet    = new Set<string>();
-  const toolSet     = new Set<string>();
-  for (const r of edgeCounts) {
-    callsMap.set(r.id, r.incoming_calls);
-    if (r.route_count > 0) routeSet.add(r.id);
-    if (r.tool_count  > 0) toolSet.add(r.id);
-  }
-
-  const scored: ScoredNode[] = [];
-
-  for (const row of symbolRows) {
-    let score = 0;
-
-    if ((callsMap.get(row.id) ?? 0) === 0) score += 3;
-
-    if (row.is_exported === 1) score += 2;
-
-    if (routeSet.has(row.id)) score += 4;
-
-    if (toolSet.has(row.id)) score += 4;
-
-    const filePath = row.file_path ?? '';
-    const filename = basename(filePath, extname(filePath)).toLowerCase();
-    if (ENTRY_FILENAMES.has(filename)) score += 2;
-
-    if (score >= 3) {
-      scored.push({
-        id: row.id,
-        name: row.name,
-        filePath: filePath,
-        language: row.language ?? 'unknown',
-        score,
-      });
-    }
-  }
-
-  // Sort by descending score so highest-priority entry points are processed first
-  return scored.sort((a, b) => b.score - a.score);
-}
-
-// ── Phase ─────────────────────────────────────────────────────────────────────
-
-export const processesPhase: PipelinePhase<ProcessesOutput> = {
-  name: 'processes',
-  deps: ['communities', 'routes', 'tools', 'structure'],
-  async execute(ctx, _deps) {
-    if (ctx.allFilesCached || !ctx.db) {
-      return {
-        processResult: {
-          processes: [],
-          memberships: new Map(),
-          stats: { totalProcesses: 0, totalSteps: 0 },
-        },
-      };
-    }
-
-    const db = ctx.db;
-
-    // Determine how many processes to create based on symbol count
-    const symbolCountRow = db
-      .prepare(
-        `SELECT COUNT(*) as cnt FROM nodes WHERE label NOT IN ('File', 'Folder', 'Community', 'Process', 'Document')`,
-      )
-      .get() as { cnt: number };
-    const symbolCount = symbolCountRow.cnt;
-    const maxProcesses = Math.max(20, Math.min(300, Math.ceil(symbolCount / 10)));
-
-    // Score and collect candidate entry points
-    const candidates = scoreEntryPoints(db);
-    const limitedCandidates = candidates.slice(0, maxProcesses);
-
-    const processNodes: MonographNode[] = [];
-    const processEdges: MonographEdge[] = [];
-    const processDefs: ProcessDef[] = [];
-    const memberships = new Map<string, string>();
-    let totalSteps = 0;
-
-    // Track which nodes are already claimed as an entry point of a process
-    // to avoid creating duplicate processes for the same entry node
-    const claimedEntries = new Set<string>();
-
-    for (const entry of limitedCandidates) {
-      if (claimedEntries.has(entry.id)) continue;
-      claimedEntries.add(entry.id);
-
-      const processNodeId = makeId('process', entry.id);
-      const stepNodeIds = bfsCallChain(entry.id, db);
-
-      const processNode: MonographNode = {
-        id: processNodeId,
-        label: 'Process',
-        name: entry.name,
-        normLabel: toNormLabel(entry.name),
-        filePath: entry.filePath,
-        language: entry.language,
-        startLine: 0,
-        endLine: 0,
-        isExported: false,
-        properties: { stepCount: stepNodeIds.size },
-      };
-      processNodes.push(processNode);
-
-      // ENTRY_POINT_OF: entry symbol → process
-      processEdges.push({
-        id: makeId(entry.id, processNodeId, 'entry_point_of'),
-        sourceId: entry.id,
-        targetId: processNodeId,
-        relation: 'ENTRY_POINT_OF',
-        confidence: 'EXTRACTED',
-        confidenceScore: 0.9,
-      });
-
-      // STEP_IN_PROCESS: process → each step symbol (including entry)
-      for (const stepNodeId of stepNodeIds) {
-        processEdges.push({
-          id: makeId(processNodeId, stepNodeId, 'step_in_process'),
-          sourceId: processNodeId,
-          targetId: stepNodeId,
-          relation: 'STEP_IN_PROCESS',
-          confidence: 'EXTRACTED',
-          confidenceScore: 0.85,
-        });
-        memberships.set(stepNodeId, processNodeId);
-      }
-
-      totalSteps += stepNodeIds.size;
-
-      processDefs.push({
-        id: processNodeId,
-        name: entry.name,
-        filePath: entry.filePath,
-        entryNodeId: entry.id,
-        stepCount: stepNodeIds.size,
-      });
-    }
-
-    insertNodes(db, processNodes);
-    insertEdges(db, processEdges);
-
-    return {
-      processResult: {
-        processes: processDefs,
-        memberships,
-        stats: {
-          totalProcesses: processDefs.length,
-          totalSteps,
-        },
-      },
-    };
-  },
-};