@thoughtflow/codebase 0.2.0

package/src/libs/codebase/graph/graph-analysis.ts

@@ -0,0 +1,231 @@
+import { DependencyGraph, DependencyEdge } from "./dependency-extractor";
+import * as path from "node:path";
+
+export interface ResolvedEdge {
+  from: string;
+  to: string;
+  resolvedPath?: string;
+  isExternal: boolean;
+  type: string;
+  lineNumber: number;
+}
+
+export class DependencyResolver {
+  resolve(graph: DependencyGraph, rootPath: string): ResolvedEdge[] {
+    const resolved: ResolvedEdge[] = [];
+
+    for (const edge of graph.edges) {
+      if (edge.to.startsWith(".") || edge.to.startsWith("/")) {
+        const fromDir = path.dirname(edge.from);
+        const candidate = path.join(fromDir, edge.to).replace(/\\/g, "/");
+        resolved.push({
+          from: edge.from,
+          to: edge.to,
+          resolvedPath: candidate,
+          isExternal: false,
+          type: edge.type,
+          lineNumber: edge.lineNumber,
+        });
+      } else {
+        resolved.push({
+          from: edge.from,
+          to: edge.to,
+          isExternal: true,
+          type: edge.type,
+          lineNumber: edge.lineNumber,
+        });
+      }
+    }
+
+    return resolved;
+  }
+
+  getAdjacencyList(graph: DependencyGraph): Map<string, Set<string>> {
+    const adj = new Map<string, Set<string>>();
+    for (const edge of graph.edges) {
+      if (!adj.has(edge.from)) adj.set(edge.from, new Set());
+      adj.get(edge.from)!.add(edge.to);
+    }
+    return adj;
+  }
+
+  getReverseAdjacency(graph: DependencyGraph): Map<string, Set<string>> {
+    const rev = new Map<string, Set<string>>();
+    for (const edge of graph.edges) {
+      if (!rev.has(edge.to)) rev.set(edge.to, new Set());
+      rev.get(edge.to)!.add(edge.from);
+    }
+    return rev;
+  }
+}
+
+export class CircularDetector {
+  findCycles(graph: DependencyGraph): string[][] {
+    const adj = new Map<string, Set<string>>();
+    const allNodes = new Set<string>();
+
+    for (const edge of graph.edges) {
+      allNodes.add(edge.from);
+      allNodes.add(edge.to);
+      if (!adj.has(edge.from)) adj.set(edge.from, new Set());
+      adj.get(edge.from)!.add(edge.to);
+    }
+
+    const index = new Map<string, number>();
+    const lowlink = new Map<string, number>();
+    const onStack = new Set<string>();
+    const stack: string[] = [];
+    const sccs: string[][] = [];
+    let idx = 0;
+
+    const strongconnect = (v: string): void => {
+      index.set(v, idx);
+      lowlink.set(v, idx);
+      idx++;
+      stack.push(v);
+      onStack.add(v);
+
+      const neighbors = adj.get(v);
+      if (neighbors) {
+        for (const w of neighbors) {
+          if (!index.has(w)) {
+            strongconnect(w);
+            lowlink.set(v, Math.min(lowlink.get(v)!, lowlink.get(w)!));
+          } else if (onStack.has(w)) {
+            lowlink.set(v, Math.min(lowlink.get(v)!, index.get(w)!));
+          }
+        }
+      }
+
+      if (lowlink.get(v) === index.get(v)) {
+        const scc: string[] = [];
+        let w: string;
+        do {
+          w = stack.pop()!;
+          onStack.delete(w);
+          scc.push(w);
+        } while (w !== v);
+
+        if (scc.length > 1) {
+          sccs.push(scc);
+        }
+      }
+    };
+
+    for (const node of allNodes) {
+      if (!index.has(node)) {
+        strongconnect(node);
+      }
+    }
+
+    return sccs;
+  }
+}
+
+export interface ImpactResult {
+  target: string;
+  hops: Array<{ distance: number; files: string[] }>;
+  totalAffectedFiles: number;
+}
+
+export class ImpactAnalyzer {
+  analyze(targetFile: string, graph: DependencyGraph, maxHops: number = 3): ImpactResult {
+    const rev = new Map<string, Set<string>>();
+
+    for (const edge of graph.edges) {
+      const key = edge.to;
+      if (!rev.has(key)) rev.set(key, new Set());
+      rev.get(key)!.add(edge.from);
+    }
+
+    const visited = new Set<string>();
+    const hops: Array<{ distance: number; files: string[] }> = [];
+    let currentLayer = [targetFile];
+    visited.add(targetFile);
+
+    for (let d = 1; d <= maxHops && currentLayer.length > 0; d++) {
+      const nextLayer: string[] = [];
+      for (const file of currentLayer) {
+        const dependents = rev.get(file);
+        if (!dependents) continue;
+        for (const dep of dependents) {
+          if (!visited.has(dep)) {
+            visited.add(dep);
+            nextLayer.push(dep);
+          }
+        }
+      }
+
+      if (nextLayer.length > 0) {
+        hops.push({ distance: d, files: nextLayer.sort() });
+      }
+      currentLayer = nextLayer;
+    }
+
+    return {
+      target: targetFile,
+      hops,
+      totalAffectedFiles: visited.size - 1,
+    };
+  }
+}
+
+export interface FlowResult {
+  entrypoint: string;
+  path: FlowStep[];
+  maxDepth: number;
+}
+
+export interface FlowStep {
+  file: string;
+  callees: string[];
+  depth: number;
+}
+
+export class CallTracer {
+  traceForward(
+    entrypoint: string,
+    graph: DependencyGraph,
+    maxDepth: number = 5
+  ): FlowResult {
+    const adj = new Map<string, Set<string>>();
+    for (const edge of graph.edges) {
+      if (!adj.has(edge.from)) adj.set(edge.from, new Set());
+      adj.get(edge.from)!.add(edge.to);
+    }
+
+    const visited = new Set<string>();
+    const path: FlowStep[] = [];
+    const queue: Array<{ file: string; depth: number }> = [{ file: entrypoint, depth: 0 }];
+    visited.add(entrypoint);
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      if (current.depth > maxDepth) continue;
+
+      const callees = adj.get(current.file);
+      const calleeList = callees ? Array.from(callees) : [];
+
+      path.push({
+        file: current.file,
+        callees: calleeList,
+        depth: current.depth,
+      });
+
+      if (callees) {
+        for (const callee of callees) {
+          if (!visited.has(callee)) {
+            visited.add(callee);
+            queue.push({ file: callee, depth: current.depth + 1 });
+          }
+        }
+      }
+    }
+
+    return {
+      entrypoint,
+      path,
+      maxDepth,
+    };
+  }
+}

package/src/libs/codebase/graph/graph-visualizer.ts

@@ -0,0 +1,124 @@
+import { DependencyGraph } from "./dependency-extractor";
+
+export interface VisualizeOptions {
+  maxNodes?: number;
+  highlightCircular?: boolean;
+  colorByLanguage?: boolean;
+  direction?: 'TD' | 'LR' | 'RL' | 'BT';
+}
+
+export class GraphVisualizer {
+  toMermaid(graph: DependencyGraph, options: VisualizeOptions = {}): string {
+    const maxNodes = options.maxNodes ?? 100;
+    const highlightCircular = options.highlightCircular ?? true;
+    const direction = options.direction ?? 'TD';
+
+    const circulars = new Set<string>();
+    if (highlightCircular) {
+      const { CircularDetector } = require("./graph-analysis");
+      const cycles = new CircularDetector().findCycles(graph);
+      for (const cycle of cycles) {
+        for (const node of cycle) {
+          circulars.add(node);
+        }
+      }
+    }
+
+    const nodeSet = new Set<string>();
+    const nodeEdges: Array<{ from: string; to: string }> = [];
+
+    for (const edge of graph.edges) {
+      if (edge.to.startsWith("http") || (!edge.to.startsWith(".") && !edge.to.startsWith("/"))) continue;
+      nodeSet.add(edge.from);
+      nodeSet.add(edge.to);
+      nodeEdges.push({ from: edge.from, to: edge.to });
+    }
+
+    const sortedNodes = Array.from(nodeSet).sort().slice(0, maxNodes);
+    const filteredEdges = nodeEdges.filter(
+      (e) => sortedNodes.includes(e.from) && sortedNodes.includes(e.to)
+    );
+
+    const lines: string[] = [`graph ${direction}`];
+    const nodeIds = new Map<string, string>();
+
+    for (const node of sortedNodes) {
+      const id = `N${sortedNodes.indexOf(node)}`;
+      nodeIds.set(node, id);
+      const shortName = node.split("/").pop()?.replace(/\.[^.]+$/, "") ?? node;
+      const label = shortName.length > 30 ? shortName.slice(0, 27) + "..." : shortName;
+      const isCircular = circulars.has(node);
+      const style = isCircular ? ":::circular" : "";
+      lines.push(`  ${id}[${JSON.stringify(label)}]${style}`);
+    }
+
+    if (highlightCircular && circulars.size > 0) {
+      lines.push(`  classDef circular fill:#ff6b6b,stroke:#c92a2a,color:white`);
+    }
+
+    for (const edge of filteredEdges) {
+      const fromId = nodeIds.get(edge.from);
+      const toId = nodeIds.get(edge.to);
+      if (fromId && toId) {
+        lines.push(`  ${fromId} --> ${toId}`);
+      }
+    }
+
+    return lines.join("\n");
+  }
+
+  toHtml(graph: DependencyGraph, title: string = "Dependency Graph"): string {
+    const nodes = new Set<string>();
+    const edges: Array<{ from: string; to: string }> = [];
+
+    for (const edge of graph.edges) {
+      if (edge.to.startsWith("http") || (!edge.to.startsWith(".") && !edge.to.startsWith("/"))) continue;
+      nodes.add(edge.from);
+      nodes.add(edge.to);
+      edges.push({ from: edge.from, to: edge.to });
+    }
+
+    const nodeArray = Array.from(nodes).sort().slice(0, 150);
+    const filteredEdges = edges.filter(
+      (e) => nodeArray.includes(e.from) && nodeArray.includes(e.to)
+    );
+
+    const nodeJson = JSON.stringify(
+      nodeArray.map((n) => ({
+        id: n,
+        label: n.split("/").pop()?.replace(/\.[^.]+$/, "") ?? n,
+      }))
+    );
+
+    const edgeJson = JSON.stringify(
+      filteredEdges.map((e) => ({ from: e.from, to: e.to }))
+    );
+
+    return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>${title}</title>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/mermaid/10.9.0/mermaid.min.js"></script>
+<style>
+  body { margin: 0; padding: 20px; font-family: monospace; background: #1e1e2e; color: #cdd6f4; }
+  #container { max-width: 100%; overflow: auto; }
+  h1 { color: #89b4fa; }
+  .stats { color: #a6adc8; font-size: 14px; margin-bottom: 20px; }
+  .mermaid { background: #181825; padding: 20px; border-radius: 8px; }
+</style>
+</head>
+<body>
+<h1>${title}</h1>
+<div class="stats">${nodeArray.length} nodes, ${filteredEdges.length} edges</div>
+<div id="container">
+<pre class="mermaid">
+${this.toMermaid(graph, { maxNodes: 150 })}
+</pre>
+</div>
+<script>mermaid.initialize({ startOnLoad: true, theme: 'dark' });</script>
+</body>
+</html>`;
+  }
+}

package/src/libs/codebase/indexer/batch-processor.ts

@@ -0,0 +1,47 @@
+export interface BatchProgress {
+  phase: string;
+  current: number;
+  total: number;
+  elapsedMs: number;
+}
+
+export class BatchProcessor {
+  private progress: BatchProgress = { phase: "idle", current: 0, total: 0, elapsedMs: 0 };
+  private startTime = 0;
+  private checkpointCallback?: (progress: BatchProgress) => Promise<void>;
+
+  onCheckpoint(cb: (progress: BatchProgress) => Promise<void>): void {
+    this.checkpointCallback = cb;
+  }
+
+  start(total: number, phase: string): void {
+    this.progress = { phase, current: 0, total, elapsedMs: 0 };
+    this.startTime = Date.now();
+  }
+
+  async advance(delta: number): Promise<void> {
+    this.progress.current = Math.min(this.progress.current + delta, this.progress.total);
+    this.progress.elapsedMs = Date.now() - this.startTime;
+    if (this.checkpointCallback && this.progress.current % 50 === 0) {
+      await this.checkpointCallback(this.progress);
+    }
+  }
+
+  getProgress(): BatchProgress {
+    return { ...this.progress };
+  }
+
+  complete(): BatchProgress {
+    this.progress.current = this.progress.total;
+    this.progress.elapsedMs = Date.now() - this.startTime;
+    return { ...this.progress };
+  }
+
+  chunk<T>(items: T[], size: number): T[][] {
+    const batches: T[][] = [];
+    for (let i = 0; i < items.length; i += size) {
+      batches.push(items.slice(i, i + size));
+    }
+    return batches;
+  }
+}

package/src/libs/codebase/indexer/chunker-markdown.ts

@@ -0,0 +1,175 @@
+import type { DiscoveredFile } from "./explorer";
+
+export interface MarkdownChunk {
+  id: string;
+  text: string;
+  startLine: number;
+  endLine: number;
+  headingLevel: number;
+  headingText: string;
+  parentHeading?: string;
+  frontmatter?: Record<string, unknown>;
+  docType?: 'guideline' | 'architecture' | 'decision' | 'business' | 'technical' | 'audit' | 'unknown';
+}
+
+export class MarkdownChunker {
+  chunkFile(file: DiscoveredFile, content: string): MarkdownChunk[] {
+    const lines = content.split("\n");
+    const frontmatter = this.parseFrontmatter(lines);
+    const contentStart = frontmatter ? frontmatter.endLine + 1 : 0;
+
+    const sections = this.splitSections(lines, contentStart);
+    const docType = this.classifyDocType(file, frontmatter?.data, content);
+
+    return sections.map((section, index) => ({
+      id: `${file.relativePath}:md:${index}`,
+      text: section.text.slice(0, 3000),
+      startLine: section.startLine + 1,
+      endLine: section.endLine + 1,
+      headingLevel: section.level,
+      headingText: section.heading,
+      parentHeading: section.parent,
+      frontmatter: frontmatter?.data,
+      docType,
+    }));
+  }
+
+  private parseFrontmatter(lines: string[]): { data: Record<string, unknown>; endLine: number } | null {
+    if (lines.length === 0 || lines[0].trim() !== "---") return null;
+
+    let endLine = -1;
+    for (let i = 1; i < lines.length; i++) {
+      if (lines[i].trim() === "---") {
+        endLine = i;
+        break;
+      }
+    }
+    if (endLine === -1) return null;
+
+    const data: Record<string, unknown> = {};
+    for (let i = 1; i < endLine; i++) {
+      const line = lines[i].trim();
+      const colonIdx = line.indexOf(":");
+      if (colonIdx === -1) continue;
+      const key = line.slice(0, colonIdx).trim();
+      const value = line.slice(colonIdx + 1).trim();
+      if (value.startsWith("[") && value.endsWith("]")) {
+        data[key] = value
+          .slice(1, -1)
+          .split(",")
+          .map((s) => s.trim().replace(/^["']|["']$/g, ""));
+      } else {
+        data[key] = value.replace(/^["']|["']$/g, "");
+      }
+    }
+
+    return { data, endLine };
+  }
+
+  private splitSections(
+    lines: string[],
+    startLine: number
+  ): Array<{ heading: string; level: number; parent?: string; text: string; startLine: number; endLine: number }> {
+    const sections: Array<{
+      heading: string;
+      level: number;
+      parent?: string;
+      text: string;
+      startLine: number;
+      endLine: number;
+    }> = [];
+
+    const headingStack: Array<{ text: string; level: number; line: number }> = [];
+    let currentContent: string[] = [];
+    let currentStart = startLine;
+    let currentHeading = "";
+    let currentLevel = 0;
+
+    for (let i = startLine; i < lines.length; i++) {
+      const line = lines[i];
+      const headingMatch = line.match(/^(#{1,6})\s+(.+)/);
+
+      if (headingMatch) {
+        if (currentContent.length > 0) {
+          sections.push({
+            heading: currentHeading || "root",
+            level: currentLevel,
+            parent: headingStack.length > 0 ? headingStack[headingStack.length - 1].text : undefined,
+            text: currentContent.join("\n"),
+            startLine: currentStart,
+            endLine: i - 1,
+          });
+        }
+
+        const level = headingMatch[1].length;
+        const headingText = headingMatch[2].trim();
+
+        while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= level) {
+          headingStack.pop();
+        }
+        headingStack.push({ text: headingText, level, line: i });
+
+        currentHeading = headingText;
+        currentLevel = level;
+        currentContent = [line];
+        currentStart = i;
+      } else {
+        currentContent.push(line);
+      }
+    }
+
+    if (currentContent.length > 0) {
+      sections.push({
+        heading: currentHeading || "root",
+        level: currentLevel,
+        parent: headingStack.length > 0 ? headingStack[headingStack.length - 1].text : undefined,
+        text: currentContent.join("\n"),
+        startLine: currentStart,
+        endLine: lines.length - 1,
+      });
+    }
+
+    return sections;
+  }
+
+  private classifyDocType(
+    file: DiscoveredFile,
+    frontmatter?: Record<string, unknown>,
+    content?: string
+  ): MarkdownChunk['docType'] {
+    const fp = file.relativePath.toLowerCase();
+    const txt = content?.toLowerCase() ?? '';
+
+    if (fp.includes('architecture') || fp.includes('arch/') || txt.includes('architecture decision')) {
+      return 'architecture';
+    }
+    if (fp.includes('decision') || fp.includes('adr') || txt.includes('decision record')) {
+      return 'decision';
+    }
+    if (fp.includes('business') || fp.includes('biznes') || fp.includes('bp-')) {
+      return 'business';
+    }
+    if (fp.includes('api') || fp.includes('endpoint') || txt.includes('api specification')) {
+      return 'technical';
+    }
+    if (fp.includes('audit') || txt.includes('audit report')) {
+      return 'audit';
+    }
+    if (fp.includes('guideline') || fp.includes('guide') || fp.includes('standard') || fp.includes('convention')) {
+      return 'guideline';
+    }
+    if (fp.includes('readme') || fp.includes('contributing') || txt.includes('documentation')) {
+      return 'technical';
+    }
+
+    const fm = frontmatter as Record<string, unknown> | undefined;
+    if (fm?.type) {
+      const t = String(fm.type).toLowerCase();
+      if (['architecture', 'decision', 'business', 'technical', 'audit', 'guideline'].includes(t)) {
+        return t as MarkdownChunk['docType'];
+      }
+    }
+
+    return 'unknown';
+  }
+}

package/src/libs/codebase/indexer/chunker.ts

@@ -0,0 +1,126 @@
+import type { DiscoveredFile } from "./explorer";
+
+export interface CodeChunk {
+  id: string;
+  text: string;
+  startLine: number;
+  endLine: number;
+  type: 'function' | 'class' | 'interface' | 'import-block' | 'export' | 'other';
+  name?: string;
+}
+
+export interface ChunkResult {
+  chunks: CodeChunk[];
+  language: string;
+  totalLines: number;
+}
+
+const LANG_MAP: Record<string, string> = {
+  ".ts": "TypeScript", ".tsx": "Tsx", ".mts": "TypeScript", ".cts": "TypeScript",
+  ".js": "JavaScript", ".jsx": "JavaScript", ".mjs": "JavaScript", ".cjs": "JavaScript",
+  ".html": "Html",
+};
+
+async function parseWithAstGrep(lang: string, content: string) {
+  const { Lang, parse } = await import("@ast-grep/napi");
+  return parse(Lang[lang as keyof typeof Lang], content);
+}
+
+const IMPORT_DQ = 'import $$$ from "$SOURCE"';
+const IMPORT_SQ = "import $$$ from '$SOURCE'";
+
+export class CodeChunker {
+  async chunkFile(file: DiscoveredFile, content: string): Promise<ChunkResult> {
+    const lang = LANG_MAP[file.extension];
+    const totalLines = content.split("\n").length;
+
+    if (!lang) {
+      return this.lineBasedFallback(file, content, totalLines);
+    }
+
+    try {
+      const root = (await parseWithAstGrep(lang, content)).root();
+      const chunks: CodeChunk[] = [];
+
+      const importNodes = [...root.findAll(IMPORT_DQ), ...root.findAll(IMPORT_SQ)];
+      if (importNodes.length > 0) {
+        const text = importNodes.map((n) => n.text()).join("\n");
+        chunks.push({
+          id: `${file.relativePath}:imports`,
+          text: text.slice(0, 2000),
+          startLine: importNodes[0].range().start.line + 1,
+          endLine: importNodes[importNodes.length - 1].range().end.line + 1,
+          type: 'import-block',
+        });
+      }
+
+      const fnRule = { rule: { kind: "function_declaration", pattern: "function $NAME" } };
+      for (const node of root.findAll(fnRule as any)) {
+        const name = node.getMatch('NAME')?.text() ?? '(anonymous)';
+        chunks.push({
+          id: `${file.relativePath}:fn:${name}`,
+          text: node.text().slice(0, 2000),
+          startLine: node.range().start.line + 1,
+          endLine: node.range().end.line + 1,
+          type: 'function',
+          name,
+        });
+      }
+
+      const clsRule = { rule: { kind: "class_declaration", pattern: "class $NAME" } };
+      for (const node of root.findAll(clsRule as any)) {
+        const name = node.getMatch('NAME')?.text() ?? '(anonymous)';
+        chunks.push({
+          id: `${file.relativePath}:class:${name}`,
+          text: node.text().slice(0, 2000),
+          startLine: node.range().start.line + 1,
+          endLine: node.range().end.line + 1,
+          type: 'class',
+          name,
+        });
+      }
+
+      if (chunks.length === 0 || this.coverageRatio(chunks, content) < 0.1) {
+        const lineChunks = this.lineBasedChunks(file, content, totalLines);
+        return { chunks: lineChunks, language: file.language ?? 'unknown', totalLines };
+      }
+
+      return { chunks, language: file.language ?? 'unknown', totalLines };
+    } catch {
+      return this.lineBasedFallback(file, content, totalLines);
+    }
+  }
+
+  private coverageRatio(chunks: CodeChunk[], content: string): number {
+    const covered = chunks.reduce((sum, c) => sum + c.text.length, 0);
+    return content.length > 0 ? covered / content.length : 0;
+  }
+
+  private lineBasedFallback(file: DiscoveredFile, content: string, totalLines: number): ChunkResult {
+    return {
+      chunks: this.lineBasedChunks(file, content, totalLines),
+      language: file.language ?? 'unknown',
+      totalLines,
+    };
+  }
+
+  private lineBasedChunks(file: DiscoveredFile, content: string, totalLines: number): CodeChunk[] {
+    const lines = content.split("\n");
+    const chunkSize = 100;
+    const chunks: CodeChunk[] = [];
+
+    for (let i = 0; i < lines.length; i += chunkSize) {
+      const end = Math.min(i + chunkSize, lines.length);
+      const text = lines.slice(i, end).join("\n");
+      chunks.push({
+        id: `${file.relativePath}:${i + 1}-${end}`,
+        text: text.slice(0, 2000),
+        startLine: i + 1,
+        endLine: end,
+        type: 'other',
+      });
+    }
+
+    return chunks;
+  }
+}