@thoughtflow/codebase 0.2.0

package/src/contexts/context-router.ts

@@ -0,0 +1,121 @@
+import type { ContextRoutingConfig, ContextDocumentCache, LlmAdapter } from "@thoughtflow/core"
+import { callStructured } from "@thoughtflow/core"
+import { z } from 'zod'
+
+const STOP_WORDS = new Set([
+  'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'shall',
+  'should', 'may', 'might', 'must', 'can', 'could', 'i', 'me', 'my',
+  'we', 'our', 'you', 'your', 'he', 'she', 'it', 'they', 'them', 'this',
+  'that', 'these', 'those', 'in', 'on', 'at', 'to', 'for', 'of', 'with',
+  'by', 'from', 'and', 'or', 'not', 'but', 'if', 'so', 'no', 'yes',
+  'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how'
+])
+
+function tokenize(text: string): Set<string> {
+  return new Set(
+    text.toLowerCase()
+      .split(/[^a-zA-Z0-9]+/)
+      .filter(w => w.length > 0 && !STOP_WORDS.has(w))
+  )
+}
+
+function computeScore(
+  promptTokens: Set<string>,
+  doc: ContextDocumentCache
+): number {
+  if (doc.keywords.length === 0) return 0
+
+  let matches = 0
+  for (const kw of doc.keywords) {
+    let hasMatch = false
+    for (const t of tokenize(kw)) {
+      if (promptTokens.has(t)) {
+        hasMatch = true
+        break
+      }
+    }
+    if (hasMatch) matches++
+  }
+
+  let categoryBonus = 0
+  if (doc.category && promptTokens.has(doc.category)) {
+    categoryBonus = 0.3
+  }
+
+  return (matches / doc.keywords.length) + categoryBonus
+}
+
+const routingSchema = z.object({
+  relevant: z.array(z.string())
+})
+
+export class ContextRouter {
+  constructor(
+    private readonly config: ContextRoutingConfig,
+    private readonly adapter: LlmAdapter,
+    private readonly model: string
+  ) {}
+
+  /**
+   * Route the given prompt against cached document metadata.
+   * Returns the list of document names that are relevant.
+   */
+  async route(
+    prompt: string,
+    documents: ContextDocumentCache[]
+  ): Promise<string[]> {
+    const promptTokens = tokenize(prompt)
+    const strategy = this.config.strategy ?? 'hybrid'
+    const threshold = this.config.relevanceThreshold ?? 0.5
+
+    // ── Step 1: keyword matching ────────────────────────────────────
+
+    const scored = documents.map(doc => ({
+      name: doc.name,
+      score: computeScore(promptTokens, doc)
+    }))
+
+    const maxScore = Math.max(...scored.map(s => s.score), 0)
+    const normalized = scored.map(s => ({
+      ...s,
+      score: maxScore > 0 ? s.score / maxScore : 0
+    }))
+
+    const keywordHits = normalized
+      .filter(s => s.score >= threshold)
+      .map(s => s.name)
+
+    if (strategy === 'keyword') {
+      return keywordHits
+    }
+
+    if (strategy === 'hybrid' && keywordHits.length > 0) {
+      return keywordHits
+    }
+
+    // ── Step 2: LLM routing ────────────────────────────────────────
+
+    if (documents.length === 0) {
+      return []
+    }
+
+    const routingModel = this.config.model?.model ?? this.model
+
+    const docLines = documents
+      .map(d => `- ${d.name}: ${d.summary}`)
+      .join('\n')
+
+    const userPrompt = `Given the user query: "${prompt}"\n\nSelect which of the following documents are relevant:\n${docLines}`
+
+    const result = await callStructured(
+      this.adapter,
+      routingModel,
+      'You are a document routing assistant. Select relevant documents based on the user query.',
+      userPrompt,
+      routingSchema
+    )
+
+    return result.result.relevant
+  }
+}

package/src/index.ts

@@ -0,0 +1,65 @@
+// Codebase index
+export * from "./libs/codebase-index.class";
+
+// Codebase libs — deep analyzer, graph, indexer, knowledge, search
+export * from "./libs/codebase/deep-analyzer";
+export * from "./libs/codebase/graph/dependency-extractor";
+export * from "./libs/codebase/graph/graph-analysis";
+export * from "./libs/codebase/graph/graph-visualizer";
+export * from "./libs/codebase/indexer/batch-processor";
+export * from "./libs/codebase/indexer/chunker-markdown";
+export * from "./libs/codebase/indexer/chunker";
+export * from "./libs/codebase/indexer/explorer";
+export * from "./libs/codebase/indexer/indexer";
+export * from "./libs/codebase/knowledge/cross-referencer";
+export * from "./libs/codebase/knowledge/markdown-extractor";
+export * from "./libs/codebase/knowledge/summarizer";
+export * from "./libs/codebase/search/bm25";
+export * from "./libs/codebase/search/hybrid-search";
+
+// Tools — codebase
+export * from "./tools/codebase/codebase-analyze.tool";
+export * from "./tools/codebase/codebase-crossref.tool";
+export * from "./tools/codebase/codebase-flow.tool";
+export * from "./tools/codebase/codebase-graph-circular.tool";
+export * from "./tools/codebase/codebase-graph-query.tool";
+export * from "./tools/codebase/codebase-graph-stats.tool";
+export * from "./tools/codebase/codebase-graph-visualize.tool";
+export * from "./tools/codebase/codebase-impact.tool";
+export * from "./tools/codebase/codebase-index.tool";
+export * from "./tools/codebase/codebase-knowledge.tool";
+export * from "./tools/codebase/codebase-search.tool";
+export * from "./tools/codebase/codebase-status.tool";
+
+// Tools — quality
+export * from "./tools/quality/quality-complexity.tool";
+export * from "./tools/quality/quality-coupling.tool";
+export * from "./tools/quality/quality-coverage.tool";
+export * from "./tools/quality/quality-deps.tool";
+export * from "./tools/quality/quality-duplication.tool";
+export * from "./tools/quality/quality-size.tool";
+export * from "./tools/quality/quality-types.tool";
+
+// Tools — neural
+export * from "./tools/neural/comparative-analysis.tool";
+export * from "./tools/neural/neural-deadcode.tool";
+export * from "./tools/neural/neural-debug.tool";
+export * from "./tools/neural/neural-explain.tool";
+export * from "./tools/neural/neural-optimize.tool";
+export * from "./tools/neural/neural-pattern.tool";
+export * from "./tools/neural/neural-review.tool";
+export * from "./tools/neural/neural-security.tool";
+export * from "./tools/neural/neural-similar.tool";
+export * from "./tools/neural/neural-smells.tool";
+export * from "./tools/neural/neural-suggest.tool";
+
+// Tools — workspace (codebase-related)
+export * from "./tools/workspace/search-code.tool";
+export * from "./tools/workspace/codebase-research.tool";
+export * from "./tools/workspace/project-scanner.tool";
+export * from "./tools/workspace/analyze-project.tool";
+
+// Contexts
+export * from "./contexts/context-decomposer";
+export * from "./contexts/context-document-manager";
+export * from "./contexts/context-router";

package/src/libs/codebase/deep-analyzer.ts

@@ -0,0 +1,287 @@
+import type { IVectorStore, IEmbeddingProvider } from "@thoughtflow/core";
+import { CodebaseExplorer, DiscoveredFile } from "./indexer/explorer";
+import { CodebaseIndexer, IndexerOptions } from "./indexer/indexer";
+import { DependencyExtractor, DependencyGraph } from "./graph/dependency-extractor";
+import { CircularDetector, ImpactAnalyzer } from "./graph/graph-analysis";
+import { GraphVisualizer } from "./graph/graph-visualizer";
+import { MarkdownExtractor, ExtractedDocument } from "./knowledge/markdown-extractor";
+import { CrossReferencer, CrossReferenceGraph } from "./knowledge/cross-referencer";
+import * as path from "node:path";
+
+export type AnalysisPhase =
+  | 'discover'
+  | 'index'
+  | 'graph'
+  | 'extract-knowledge'
+  | 'summarize'
+  | 'cross-reference'
+  | 'complete';
+
+export interface DeepAnalysisConfig {
+  repoPath: string;
+  vectorStore: IVectorStore;
+  embeddingProvider: IEmbeddingProvider;
+  phases?: AnalysisPhase[];
+  onProgress?: (phase: AnalysisPhase, message: string, percent: number) => void;
+}
+
+export interface DeepAnalysisReport {
+  repoName: string;
+  repoPath: string;
+  analyzedAt: string;
+  durationMs: number;
+  stats: {
+    totalFiles: number;
+    codeFiles: number;
+    docFiles: number;
+    configFiles: number;
+    testFiles: number;
+    totalChunks: number;
+  };
+  architecture: {
+    techStack: string[];
+    topModules: string[];
+    entryPoints: string[];
+    circularDependencies: string[][];
+  };
+  knowledge: {
+    documentsExtracted: number;
+    usefulDocuments: number;
+    noiseDocuments: number;
+    decisions: number;
+    qualityScore: number;
+  };
+  traceability: {
+    totalReferences: number;
+    featuresWithCode: number;
+    featuresWithoutCode: number;
+    orphanDocuments: string[];
+    undocumentedModules: string[];
+  };
+  summary: string;
+  recommendations: string[];
+  mermaidDiagram: string;
+}
+
+export class DeepAnalyzer {
+  private config: DeepAnalysisConfig;
+  private startTime = 0;
+
+  constructor(config: DeepAnalysisConfig) {
+    this.config = {
+      phases: ['discover', 'index', 'graph', 'extract-knowledge', 'summarize', 'cross-reference', 'complete'],
+      ...config,
+    };
+  }
+
+  async analyze(): Promise<DeepAnalysisReport> {
+    this.startTime = Date.now();
+    const phases = this.config.phases!;
+    let files: DiscoveredFile[] = [];
+    let graph: DependencyGraph = { nodes: new Map(), edges: [] };
+    let documents: ExtractedDocument[] = [];
+    let crossRefGraph: CrossReferenceGraph = { nodes: new Map(), edges: [] };
+    let mermaidDiagram = "";
+
+    if (phases.includes('discover')) {
+      this.emit('discover', 'Discovering files...', 0);
+      const explorer = new CodebaseExplorer();
+      files = explorer.explore({ rootPath: this.config.repoPath });
+      this.emit('discover', `Found ${files.length} files`, 100);
+    }
+
+    if (phases.includes('index')) {
+      this.emit('index', 'Indexing codebase...', 0);
+      const indexer = new CodebaseIndexer(
+        this.config.vectorStore,
+        this.config.embeddingProvider,
+        { projectPath: this.config.repoPath, onProgress: (p) => {
+          this.emit('index', `${p.phase}: ${p.chunksEmbedded}/${p.chunksCreated} chunks`,
+            p.chunksCreated > 0 ? (p.chunksEmbedded / p.chunksCreated) * 100 : 50);
+        }}
+      );
+      await indexer.index();
+      this.emit('index', 'Indexing complete', 100);
+    }
+
+    if (phases.includes('graph')) {
+      this.emit('graph', 'Building dependency graph...', 0);
+      if (files.length === 0) {
+        const explorer = new CodebaseExplorer();
+        files = explorer.explore({ rootPath: this.config.repoPath });
+      }
+
+      const codeFiles = files.filter(
+        (f) => f.classification === 'source-code' || f.classification === 'test'
+      ).map((f) => ({ relativePath: f.relativePath, absolutePath: f.absolutePath }));
+
+      const depExtractor = new DependencyExtractor();
+      graph = await depExtractor.buildGraph(codeFiles, this.config.repoPath);
+      this.emit('graph', `Built graph: ${graph.nodes.size} nodes, ${graph.edges.length} edges`, 100);
+
+      const visualizer = new GraphVisualizer();
+      mermaidDiagram = visualizer.toMermaid(graph, { maxNodes: 100 });
+    }
+
+    if (phases.includes('extract-knowledge')) {
+      this.emit('extract-knowledge', 'Extracting knowledge from documentation...', 0);
+      if (files.length === 0) {
+        const explorer = new CodebaseExplorer();
+        files = explorer.explore({ rootPath: this.config.repoPath });
+      }
+
+      const docFiles = files.filter((f) => f.classification === 'documentation');
+      const extractor = new MarkdownExtractor();
+      documents = docFiles.map((f) => extractor.extract(f.absolutePath));
+      this.emit('extract-knowledge', `Extracted ${documents.length} documents`, 100);
+    }
+
+    if (phases.includes('cross-reference')) {
+      this.emit('cross-reference', 'Building cross-references...', 0);
+      const crossRef = new CrossReferencer();
+      const codeFiles = files.filter(
+        (f) => f.classification === 'source-code'
+      ).map((f) => ({ relativePath: f.relativePath }));
+
+      crossRefGraph = crossRef.buildReferences(documents, codeFiles, graph.edges as any);
+      this.emit('cross-reference', `Built ${crossRefGraph.edges.length} references`, 100);
+    }
+
+    // Build report
+    this.emit('complete', 'Generating report...', 90);
+
+    const cycles = new CircularDetector().findCycles(graph);
+    const usefulDocs = documents.filter((d) => d.qualityScore >= 0.3);
+    const noiseDocs = documents.filter((d) => d.qualityScore < 0.3);
+    const decisions = documents.filter((d) => d.docType === 'decision');
+
+    const techStack = this.detectTechStack(files, graph);
+
+    const report: DeepAnalysisReport = {
+      repoName: path.basename(this.config.repoPath),
+      repoPath: this.config.repoPath,
+      analyzedAt: new Date().toISOString(),
+      durationMs: Date.now() - this.startTime,
+      stats: {
+        totalFiles: files.length,
+        codeFiles: files.filter((f) => f.classification === 'source-code').length,
+        docFiles: files.filter((f) => f.classification === 'documentation').length,
+        configFiles: files.filter((f) => f.classification === 'config').length,
+        testFiles: files.filter((f) => f.classification === 'test').length,
+        totalChunks: 0,
+      },
+      architecture: {
+        techStack,
+        topModules: this.findTopModules(graph, 10),
+        entryPoints: this.findEntryPoints(graph),
+        circularDependencies: cycles,
+      },
+      knowledge: {
+        documentsExtracted: documents.length,
+        usefulDocuments: usefulDocs.length,
+        noiseDocuments: noiseDocs.length,
+        decisions: decisions.length,
+        qualityScore: documents.length > 0
+          ? usefulDocs.reduce((sum, d) => sum + d.qualityScore, 0) / usefulDocs.length
+          : 0,
+      },
+      traceability: {
+        totalReferences: crossRefGraph.edges.length,
+        featuresWithCode: 0,
+        featuresWithoutCode: 0,
+        orphanDocuments: crossRefGraph ? (new CrossReferencer() as any).findOrphanDocuments?.(crossRefGraph) ?? [] : [],
+        undocumentedModules: crossRefGraph ? (new CrossReferencer() as any).findUndocumentedCode?.(crossRefGraph) ?? [] : [],
+      },
+      summary: `This repository contains ${files.length} files across ${techStack.join(", ")} technologies.`,
+      recommendations: this.generateRecommendations(graph, cycles, usefulDocs, noiseDocs),
+      mermaidDiagram,
+    };
+
+    this.emit('complete', 'Analysis complete', 100);
+    return report;
+  }
+
+  private emit(phase: AnalysisPhase, message: string, percent: number): void {
+    this.config.onProgress?.(phase, message, percent);
+  }
+
+  private detectTechStack(files: DiscoveredFile[], graph: DependencyGraph): string[] {
+    const tech: string[] = [];
+    const langs = new Set<string>();
+    const configPaths = new Set<string>();
+
+    for (const file of files) {
+      if (file.language) langs.add(file.language);
+      configPaths.add(file.relativePath.toLowerCase());
+    }
+
+    if (langs.has("typescript") || langs.has("javascript")) tech.push("TypeScript");
+    if (langs.has("python")) tech.push("Python");
+    if (langs.has("go")) tech.push("Go");
+    if (langs.has("rust")) tech.push("Rust");
+    if (langs.has("java")) tech.push("Java");
+
+    const pathStr = Array.from(configPaths).join(" ");
+    if (pathStr.includes("package.json")) tech.push("Node.js");
+    if (pathStr.includes("docker")) tech.push("Docker");
+    if (pathStr.includes("kubernetes") || pathStr.includes("k8s")) tech.push("Kubernetes");
+    if (pathStr.includes("prisma")) tech.push("Prisma");
+    if (pathStr.includes(".github/workflows")) tech.push("GitHub Actions");
+    if (pathStr.includes("angular.json")) tech.push("Angular");
+    if (pathStr.includes("nest-cli.json") || pathStr.includes("@nestjs")) tech.push("NestJS");
+    if (pathStr.includes("next.config")) tech.push("Next.js");
+    if (pathStr.includes("react")) tech.push("React");
+
+    return tech;
+  }
+
+  private findTopModules(graph: DependencyGraph, limit: number): string[] {
+    const dependencyCount = new Map<string, number>();
+    for (const edge of graph.edges) {
+      if (!edge.to.startsWith(".")) continue;
+      dependencyCount.set(edge.to, (dependencyCount.get(edge.to) ?? 0) + 1);
+    }
+    return Array.from(dependencyCount.entries())
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, limit)
+      .map(([mod]) => mod);
+  }
+
+  private findEntryPoints(graph: DependencyGraph): string[] {
+    const hasIncomingEdge = new Set<string>();
+    for (const edge of graph.edges) {
+      hasIncomingEdge.add(edge.to);
+    }
+    return Array.from(graph.nodes.keys()).filter((n) => !hasIncomingEdge.has(n)).slice(0, 20);
+  }
+
+  private generateRecommendations(
+    graph: DependencyGraph,
+    cycles: string[][],
+    usefulDocs: ExtractedDocument[],
+    noiseDocs: ExtractedDocument[]
+  ): string[] {
+    const recs: string[] = [];
+
+    if (cycles.length > 0) {
+      recs.push(`Found ${cycles.length} circular dependency group(s) — review and break cycles`);
+    }
+
+    if (noiseDocs.length > usefulDocs.length) {
+      recs.push(`High documentation noise ratio (${noiseDocs.length} low-quality vs ${usefulDocs.length} useful docs) — consider cleanup`);
+    }
+
+    const entryPoints = this.findEntryPoints(graph);
+    if (entryPoints.length === 0 && graph.nodes.size > 10) {
+      recs.push("No clear entry points detected — consider documenting the application entry");
+    }
+
+    const totalEdges = graph.edges.length;
+    const externalEdges = graph.edges.filter((e) => !e.to.startsWith(".")).length;
+    if (externalEdges > totalEdges * 0.5) {
+      recs.push("High external dependency ratio — review for potential vendor lock-in");
+    }
+
+    return recs;
+  }
+}

package/src/libs/codebase/graph/dependency-extractor.ts

@@ -0,0 +1,165 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export interface DependencyEdge {
+  from: string;
+  to: string;
+  type: 'import' | 'require' | 'dynamic-import';
+  symbols?: string[];
+  lineNumber: number;
+}
+
+export interface DependencyGraph {
+  nodes: Map<string, string[]>;
+  edges: DependencyEdge[];
+}
+
+const LANG_MAP: Record<string, string> = {
+  ".ts": "TypeScript", ".tsx": "Tsx", ".mts": "TypeScript", ".cts": "TypeScript",
+  ".js": "JavaScript", ".jsx": "JavaScript", ".mjs": "JavaScript", ".cjs": "JavaScript",
+  ".html": "Html",
+};
+
+async function parseWithAstGrep(lang: string, content: string) {
+  const { Lang, parse } = await import("@ast-grep/napi");
+  return parse(Lang[lang as keyof typeof Lang], content);
+}
+
+export class DependencyExtractor {
+  async extractImports(filePath: string, content: string): Promise<DependencyEdge[]> {
+    const ext = path.extname(filePath).toLowerCase();
+    const lang = LANG_MAP[ext];
+    if (!lang) return [];
+
+    try {
+      const root = (await parseWithAstGrep(lang, content)).root();
+      const edges: DependencyEdge[] = [];
+
+      const namedPatterns = [
+        'import { $NAME } from "$SOURCE"',
+        "import { $NAME } from '$SOURCE'",
+      ];
+
+      const defaultPatterns = [
+        'import $NAME from "$SOURCE"',
+        "import $NAME from '$SOURCE'",
+      ];
+
+      for (const pattern of namedPatterns) {
+        for (const node of root.findAll(pattern)) {
+          const source = node.getMatch('SOURCE')?.text();
+          if (!source) continue;
+
+          const cleanSource = source.replace(/^["']|["']$/g, "");
+          if (!cleanSource) continue;
+
+          const fullText = node.text();
+          const braceMatch = fullText.match(/\{([^}]+)\}/);
+          const symbols: string[] = [];
+          if (braceMatch) {
+            const names = braceMatch[1].match(/[A-Za-z_$][\w$]*/g) ?? [];
+            symbols.push(...names);
+          }
+
+          if (symbols.length === 0) {
+            const name = node.getMatch('NAME')?.text();
+            if (name) symbols.push(name);
+          }
+
+          edges.push({
+            from: filePath,
+            to: cleanSource,
+            type: 'import',
+            symbols: symbols.length > 0 ? symbols : undefined,
+            lineNumber: node.range().start.line + 1,
+          });
+        }
+      }
+
+      for (const pattern of defaultPatterns) {
+        for (const node of root.findAll(pattern)) {
+          const source = node.getMatch('SOURCE')?.text();
+          if (!source) continue;
+
+          const cleanSource = source.replace(/^["']|["']$/g, "");
+          if (!cleanSource) continue;
+
+          const name = node.getMatch('NAME')?.text();
+          edges.push({
+            from: filePath,
+            to: cleanSource,
+            type: 'import',
+            symbols: name ? [name] : undefined,
+            lineNumber: node.range().start.line + 1,
+          });
+        }
+      }
+
+      for (const pattern of ["const $NAME = require(\"$SOURCE\")", "const $NAME = require('$SOURCE')"]) {
+        for (const node of root.findAll(pattern)) {
+          const source = node.getMatch('SOURCE')?.text();
+          if (!source) continue;
+
+          const cleanSource = source.replace(/^["']|["']$/g, "");
+          if (!cleanSource) continue;
+
+          edges.push({
+            from: filePath,
+            to: cleanSource,
+            type: 'require',
+            lineNumber: node.range().start.line + 1,
+          });
+        }
+      }
+
+      return edges;
+    } catch {
+      return [];
+    }
+  }
+
+  async buildGraph(files: Array<{ relativePath: string; absolutePath: string }>, rootPath: string): Promise<DependencyGraph> {
+    const nodes = new Map<string, string[]>();
+    const edges: DependencyEdge[] = [];
+
+    for (const file of files) {
+      let content: string;
+      try {
+        content = fs.readFileSync(file.absolutePath, "utf8");
+      } catch {
+        continue;
+      }
+
+      const fileEdges = await this.extractImports(file.relativePath, content);
+      for (const edge of fileEdges) {
+        const resolved = this.resolvePath(edge.to, file.relativePath, rootPath);
+        edge.to = resolved;
+        edges.push(edge);
+
+        if (!nodes.has(edge.from)) {
+          nodes.set(edge.from, []);
+        }
+        nodes.get(edge.from)!.push(resolved);
+      }
+    }
+
+    return { nodes, edges };
+  }
+
+  private resolvePath(importPath: string, fromFile: string, rootPath: string): string {
+    if (!importPath.startsWith(".")) return importPath;
+
+    const fromDir = path.dirname(fromFile);
+    const resolved = path.join(fromDir, importPath).replace(/\\/g, "/");
+
+    const extensions = [".ts", ".tsx", ".js", ".jsx", ".mts", ".mjs", "/index.ts", "/index.js"];
+    for (const ext of extensions) {
+      const fullPath = path.join(rootPath, resolved + ext);
+      if (fs.existsSync(fullPath)) {
+        return resolved + ext;
+      }
+    }
+
+    return resolved;
+  }
+}