@thoughtflow/codebase 0.2.0

package/src/libs/codebase/indexer/explorer.ts

@@ -0,0 +1,269 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { ResourceKind } from "@thoughtflow/core";
+export type FileClassification =
+  | "source-code"
+  | "documentation"
+  | "config"
+  | "data"
+  | "test"
+  | "infrastructure"
+  | "schema"
+  | "api-spec"
+  | "unknown";
+
+export interface ExplorerOptions {
+  rootPath: string;
+  ignorePatterns?: string[];
+  respectGitignore?: boolean;
+  includeExtensions?: string[];
+  excludeExtensions?: string[];
+  maxFileSizeBytes?: number;
+}
+
+export interface DiscoveredFile {
+  absolutePath: string;
+  relativePath: string;
+  extension: string;
+  sizeBytes: number;
+  language?: string;
+  classification: FileClassification;
+  resourceKind?: ResourceKind;
+}
+
+const DEFAULT_SKIP_DIRS = new Set([
+  "node_modules",
+  ".git",
+  "dist",
+  "build",
+  "coverage",
+  "out",
+  ".next",
+  ".turbo",
+  ".cache",
+  "vendor",
+  "__pycache__",
+  ".venv",
+  "venv",
+  ".idea",
+  ".vscode",
+  "target",
+  ".mypy_cache",
+  ".pytest_cache",
+]);
+
+const DEFAULT_BINARY_EXTS = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".svg",
+  ".pdf", ".zip", ".gz", ".tar", ".tgz", ".bz2", ".7z", ".rar",
+  ".mp4", ".mov", ".avi", ".mp3", ".wav", ".ogg",
+  ".woff", ".woff2", ".ttf", ".eot", ".otf",
+  ".class", ".jar", ".exe", ".dll", ".so", ".dylib", ".bin", ".wasm", ".map",
+  ".pyc", ".pyo", ".o", ".a", ".lib", ".obj",
+]);
+
+const DOC_EXTS = new Set([".md", ".mdx", ".rst", ".txt", ".adoc", ".wiki"]);
+const CONFIG_EXTS = new Set([".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env", ".rc"]);
+const SCHEMA_EXTS = new Set([".prisma", ".graphql", ".gql", ".proto", ".sql"]);
+const INFRA_EXTS = new Set([".tf", ".hcl", ".bicep", ".dockerfile"]);
+const TEST_PATTERNS = [/\.test\./, /\.spec\./, /\.e2e/, /\.integration\./, /__tests__\//, /test\//, /tests\//, /cypress\//];
+
+const EXTENSION_LANG_MAP: Record<string, string> = {
+  ".ts": "typescript",
+  ".tsx": "tsx",
+  ".mts": "typescript",
+  ".cts": "typescript",
+  ".js": "javascript",
+  ".jsx": "javascript",
+  ".mjs": "javascript",
+  ".cjs": "javascript",
+  ".py": "python",
+  ".pyi": "python",
+  ".go": "go",
+  ".rs": "rust",
+  ".java": "java",
+  ".kt": "kotlin",
+  ".swift": "swift",
+  ".rb": "ruby",
+  ".php": "php",
+  ".cs": "csharp",
+  ".c": "c",
+  ".cpp": "cpp",
+  ".h": "c",
+  ".scala": "scala",
+  ".lua": "lua",
+  ".sh": "bash",
+  ".bash": "bash",
+  ".html": "html",
+  ".css": "css",
+  ".scss": "scss",
+  ".less": "less",
+};
+
+export class CodebaseExplorer {
+  explore(options: ExplorerOptions): DiscoveredFile[] {
+    const root = path.resolve(options.rootPath);
+    const maxSize = options.maxFileSizeBytes ?? 512 * 1024;
+    const ignorePatterns = options.ignorePatterns ?? [];
+    const respectGitignore = options.respectGitignore ?? true;
+
+    const gitignorePatterns = respectGitignore ? this.parseGitignores(root) : [];
+
+    const files: DiscoveredFile[] = [];
+    this.walk(root, root, files, maxSize, gitignorePatterns, ignorePatterns, options);
+    return files;
+  }
+
+  private walk(
+    dir: string,
+    root: string,
+    out: DiscoveredFile[],
+    maxSize: number,
+    gitignorePatterns: RegExp[],
+    extraIgnores: string[],
+    options: ExplorerOptions,
+  ): void {
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const abs = path.join(dir, entry.name);
+      const rel = path.relative(root, abs).replace(/\\/g, "/");
+
+      if (entry.isDirectory()) {
+        if (DEFAULT_SKIP_DIRS.has(entry.name)) continue;
+        if (entry.name.startsWith(".") && !options.includeExtensions?.length) continue;
+        if (this.isIgnored(rel, gitignorePatterns, extraIgnores)) continue;
+        this.walk(abs, root, out, maxSize, gitignorePatterns, extraIgnores, options);
+        continue;
+      }
+
+      if (!entry.isFile()) continue;
+
+      const ext = path.extname(abs).toLowerCase();
+      if (DEFAULT_BINARY_EXTS.has(ext)) continue;
+      if (this.isIgnored(rel, gitignorePatterns, extraIgnores)) continue;
+
+      if (options.excludeExtensions?.includes(ext)) continue;
+
+      let stat: fs.Stats;
+      try {
+        stat = fs.statSync(abs);
+      } catch {
+        continue;
+      }
+      if (stat.size > maxSize) continue;
+      if (!this.isLikelyText(abs)) continue;
+
+      const classification = this.classify(rel, ext);
+      const resourceKind = this.toResourceKind(classification);
+
+      out.push({
+        absolutePath: abs,
+        relativePath: rel,
+        extension: ext,
+        sizeBytes: stat.size,
+        language: EXTENSION_LANG_MAP[ext],
+        classification,
+        resourceKind,
+      });
+    }
+  }
+
+  private classify(relativePath: string, ext: string): FileClassification {
+    if (DOC_EXTS.has(ext)) return "documentation";
+    if (SCHEMA_EXTS.has(ext)) return "schema";
+    if (INFRA_EXTS.has(ext)) return "infrastructure";
+    if (relativePath.includes("docker-compose") || relativePath.includes("Dockerfile") || relativePath.includes("Makefile")) return "infrastructure";
+    if (relativePath.includes("openapi") || relativePath.includes("swagger")) return "api-spec";
+    if (relativePath.includes(".github/workflows") || relativePath.includes(".gitlab-ci")) return "infrastructure";
+    if (CONFIG_EXTS.has(ext)) return "config";
+    if (TEST_PATTERNS.some((p) => p.test(relativePath))) return "test";
+    if (EXTENSION_LANG_MAP[ext]) return "source-code";
+    if (ext === ".json" || ext === ".yaml" || ext === ".yml") return "config";
+    return "unknown";
+  }
+
+  private toResourceKind(classification: FileClassification): ResourceKind | undefined {
+    switch (classification) {
+      case "schema":
+        return "schema";
+      case "config":
+        return "config";
+      case "infrastructure":
+        return "infra";
+      case "api-spec":
+        return "api-spec";
+      default:
+        return undefined;
+    }
+  }
+
+  private parseGitignores(root: string): RegExp[] {
+    const patterns: RegExp[] = [];
+    const gitignorePath = path.join(root, ".gitignore");
+    try {
+      const content = fs.readFileSync(gitignorePath, "utf8");
+      for (const line of content.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith("#")) continue;
+        if (trimmed.startsWith("/")) continue;
+        try {
+          patterns.push(new RegExp(
+            trimmed
+              .replace(/\./g, "\\.")
+              .replace(/\*\*/g, "§§DOUBLESTAR§§")
+              .replace(/\*/g, "[^/]*")
+              .replace(/§§DOUBLESTAR§§/g, ".*")
+              .replace(/\?/g, "."),
+          ));
+        } catch {
+          /* noop */
+        }
+      }
+    } catch {
+      /* noop */
+    }
+    return patterns;
+  }
+
+  private isIgnored(relPath: string, gitignorePatterns: RegExp[], extras: string[]): boolean {
+    for (const p of gitignorePatterns) {
+      if (p.test(relPath)) return true;
+    }
+    for (const p of extras) {
+      try {
+        if (new RegExp(p).test(relPath)) return true;
+      } catch {
+        /* noop */
+      }
+    }
+    return false;
+  }
+
+  private isLikelyText(absPath: string): boolean {
+    let fd: number | null = null;
+    try {
+      fd = fs.openSync(absPath, "r");
+      const sample = Buffer.allocUnsafe(1024);
+      const read = fs.readSync(fd, sample, 0, sample.length, 0);
+      for (let i = 0; i < read; i++) {
+        if (sample[i] === 0) return false;
+      }
+      return true;
+    } catch {
+      return false;
+    } finally {
+      if (fd !== null) {
+        try {
+          fs.closeSync(fd);
+        } catch {
+          /* noop */
+        }
+      }
+    }
+  }
+}

package/src/libs/codebase/indexer/indexer.ts

@@ -0,0 +1,174 @@
+import type { IVectorStore, IEmbeddingProvider, VectorChunk, ChunkMetadata } from "@thoughtflow/core";
+import { CodebaseExplorer, DiscoveredFile } from "./explorer";
+import { CodeChunker, CodeChunk } from "./chunker";
+import { MarkdownChunker, MarkdownChunk } from "./chunker-markdown";
+import { BatchProcessor } from "./batch-processor";
+import * as fs from "node:fs";
+
+export interface IndexProgress {
+  phase: "discovering" | "chunking" | "embedding" | "storing" | "complete";
+  filesDiscovered: number;
+  filesProcessed: number;
+  chunksCreated: number;
+  chunksEmbedded: number;
+  elapsedMs: number;
+}
+
+export interface IndexResult {
+  totalFiles: number;
+  totalChunks: number;
+  timeMs: number;
+  collections: string[];
+}
+
+export interface IndexerOptions {
+  projectPath: string;
+  batchSize?: number;
+  includeDocs?: boolean;
+  onProgress?: (p: IndexProgress) => void;
+}
+
+export class CodebaseIndexer {
+  private explorer: CodebaseExplorer;
+  private chunker: CodeChunker;
+  private mdChunker: MarkdownChunker;
+  private processor: BatchProcessor;
+  private progress: IndexProgress;
+
+  constructor(
+    private vectorStore: IVectorStore,
+    private embeddingProvider: IEmbeddingProvider,
+    private options: IndexerOptions
+  ) {
+    this.explorer = new CodebaseExplorer();
+    this.chunker = new CodeChunker();
+    this.mdChunker = new MarkdownChunker();
+    this.processor = new BatchProcessor();
+    this.progress = {
+      phase: "discovering",
+      filesDiscovered: 0,
+      filesProcessed: 0,
+      chunksCreated: 0,
+      chunksEmbedded: 0,
+      elapsedMs: 0,
+    };
+  }
+
+  async index(): Promise<IndexResult> {
+    const startTime = Date.now();
+
+    await this.vectorStore.initialize?.();
+    await this.embeddingProvider.initialize?.();
+
+    this.updateProgress("discovering");
+    const files = this.explorer.explore({ rootPath: this.options.projectPath });
+    this.progress.filesDiscovered = files.length;
+    this.updateProgress("chunking");
+
+    const batchSize = this.options.batchSize ?? 50;
+    this.processor.start(files.length, "chunking");
+
+    const allChunks: Array<{ chunk: CodeChunk | MarkdownChunk; file: DiscoveredFile; isMarkdown: boolean }> = [];
+
+    for (const batch of this.processor.chunk(files, batchSize)) {
+      for (const file of batch) {
+        let content: string;
+        try {
+          content = fs.readFileSync(file.absolutePath, "utf8");
+        } catch {
+          continue;
+        }
+
+        if (file.classification === "documentation" && this.options.includeDocs !== false) {
+          const mdChunks = this.mdChunker.chunkFile(file, content);
+          for (const c of mdChunks) {
+            allChunks.push({ chunk: c, file, isMarkdown: true });
+          }
+        } else if (file.classification === "source-code" || file.classification === "test") {
+          const result = await this.chunker.chunkFile(file, content);
+          for (const c of result.chunks) {
+            allChunks.push({ chunk: c, file, isMarkdown: false });
+          }
+        }
+
+        this.progress.filesProcessed++;
+        this.progress.chunksCreated = allChunks.length;
+      }
+      await this.processor.advance(batch.length);
+    }
+
+    if (allChunks.length === 0) {
+      return {
+        totalFiles: files.length,
+        totalChunks: 0,
+        timeMs: Date.now() - startTime,
+        collections: [],
+      };
+    }
+
+    this.updateProgress("embedding");
+    this.processor.start(allChunks.length, "embedding");
+    const texts = allChunks.map((c) => c.chunk.text.slice(0, 512));
+    const vectors: number[][] = [];
+    const embBatchSize = this.embeddingProvider.maxBatchSize;
+
+    for (const batch of this.processor.chunk(texts, embBatchSize)) {
+      const batchVectors = await this.embeddingProvider.embed(batch);
+      vectors.push(...batchVectors);
+      this.progress.chunksEmbedded = vectors.length;
+      await this.processor.advance(batch.length);
+    }
+
+    this.updateProgress("storing");
+    this.processor.start(allChunks.length, "storing");
+
+    const vectorChunks: VectorChunk[] = allChunks.map((item, i) => {
+      const c = item.chunk;
+      const metadata: ChunkMetadata = {
+        projectPath: this.options.projectPath,
+        filePath: item.file.relativePath,
+        startLine: c.startLine,
+        endLine: c.endLine,
+        language: item.file.language,
+        chunkType: item.file.classification === "documentation" ? "documentation" : "code",
+        resourceKind: item.file.resourceKind,
+        tags: [],
+      };
+      if (item.isMarkdown) {
+        const mdChunk = c as MarkdownChunk;
+        if (mdChunk.docType) {
+          metadata.category = mdChunk.docType;
+        }
+      }
+      return {
+        id: c.id,
+        vector: vectors[i] ?? new Array(this.embeddingProvider.dimensions).fill(0),
+        text: c.text,
+        metadata,
+      };
+    });
+
+    const storeBatchSize = 500;
+    for (const batch of this.processor.chunk(vectorChunks, storeBatchSize)) {
+      await this.vectorStore.insert(batch);
+      await this.processor.advance(batch.length);
+    }
+
+    this.updateProgress("complete");
+    const elapsed = Date.now() - startTime;
+
+    await this.embeddingProvider.dispose?.();
+
+    return {
+      totalFiles: files.length,
+      totalChunks: allChunks.length,
+      timeMs: elapsed,
+      collections: ["default"],
+    };
+  }
+
+  private updateProgress(phase: IndexProgress["phase"]): void {
+    this.progress.phase = phase;
+    this.options.onProgress?.({ ...this.progress });
+  }
+}

package/src/libs/codebase/knowledge/cross-referencer.ts

@@ -0,0 +1,182 @@
+import { ExtractedDocument } from "./markdown-extractor";
+
+export interface CrossReference {
+  sourceId: string;
+  targetId: string;
+  type: 'implements' | 'documents' | 'relates-to' | 'depends-on' | 'decision-for';
+  confidence: number;
+  evidence: string;
+}
+
+export interface CrossRefNode {
+  id: string;
+  type: 'feature' | 'code' | 'document' | 'decision' | 'pattern';
+  label: string;
+  filePath?: string;
+  summary?: string;
+}
+
+export interface CrossReferenceGraph {
+  nodes: Map<string, CrossRefNode>;
+  edges: CrossReference[];
+}
+
+export class CrossReferencer {
+  buildReferences(
+    documents: ExtractedDocument[],
+    codeFiles: Array<{ relativePath: string; content?: string }>,
+    dependencyEdges?: Array<{ from: string; to: string }>
+  ): CrossReferenceGraph {
+    const nodes = new Map<string, CrossRefNode>();
+    const edges: CrossReference[] = [];
+
+    for (const doc of documents) {
+      nodes.set(doc.id, {
+        id: doc.id,
+        type: 'document',
+        label: doc.title,
+        filePath: doc.filePath,
+        summary: doc.summary,
+      });
+
+      if (doc.docType === 'decision') {
+        nodes.set(doc.id, { ...nodes.get(doc.id)!, type: 'decision' });
+      }
+    }
+
+    for (const file of codeFiles) {
+      const id = file.relativePath.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
+      nodes.set(id, {
+        id,
+        type: 'code',
+        label: file.relativePath,
+        filePath: file.relativePath,
+      });
+    }
+
+    for (const doc of documents) {
+      for (const ref of doc.references) {
+        const normalizedRef = ref.replace(/\.md$/, "").replace(/[^a-z0-9]+/gi, "-").toLowerCase();
+        const targetNode = nodes.get(normalizedRef);
+        if (targetNode) {
+          edges.push({
+            sourceId: doc.id,
+            targetId: normalizedRef,
+            type: 'relates-to',
+            confidence: 0.7,
+            evidence: `Explicit reference in ${doc.filePath}`,
+          });
+        }
+      }
+
+      for (const codeRef of doc.codeReferences) {
+        const matchingCodeNode = Array.from(nodes.values()).find(
+          (n) => n.type === 'code' && n.filePath?.includes(codeRef.replace(/^\.\//, ""))
+        );
+        if (matchingCodeNode) {
+          edges.push({
+            sourceId: doc.id,
+            targetId: matchingCodeNode.id,
+            type: 'documents',
+            confidence: 0.5,
+            evidence: `Code reference in ${doc.filePath}`,
+          });
+        }
+      }
+
+      const docDir = doc.filePath.replace(/\.[^.]+$/, "").toLowerCase();
+      for (const [nodeId, node] of nodes) {
+        if (node.type === 'code' && node.filePath) {
+          const codeDir = node.filePath.toLowerCase();
+          if (codeDir.includes(docDir.split("/").pop() ?? "") || docDir.includes(codeDir.split("/").pop() ?? "")) {
+            edges.push({
+              sourceId: doc.id,
+              targetId: nodeId,
+              type: 'relates-to',
+              confidence: 0.3,
+              evidence: `Path similarity: ${doc.filePath} ↔ ${node.filePath}`,
+            });
+          }
+        }
+      }
+    }
+
+    if (dependencyEdges) {
+      for (const edge of dependencyEdges) {
+        const fromId = edge.from.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
+        const toId = edge.to.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
+
+        const fromNode = Array.from(nodes.values()).find((n) => n.filePath === edge.from);
+        const toNode = Array.from(nodes.values()).find((n) => n.filePath === edge.to);
+
+        if (fromNode) {
+          nodes.set(fromId, fromNode);
+        }
+        if (toNode) {
+          nodes.set(toId, toNode);
+        }
+
+        edges.push({
+          sourceId: fromId,
+          targetId: toId,
+          type: 'depends-on',
+          confidence: 1.0,
+          evidence: `Import: ${edge.from} → ${edge.to}`,
+        });
+      }
+    }
+
+    return { nodes, edges };
+  }
+
+  findOrphanDocuments(graph: CrossReferenceGraph): string[] {
+    const referenced = new Set(graph.edges.map((e) => e.sourceId));
+    return Array.from(graph.nodes.values())
+      .filter((n) => n.type === 'document' && !referenced.has(n.id))
+      .map((n) => n.filePath ?? n.id);
+  }
+
+  findUndocumentedCode(
+    graph: CrossReferenceGraph,
+    minImportance: number = 1
+  ): string[] {
+    const documentedCode = new Set(
+      graph.edges
+        .filter((e) => e.type === 'documents')
+        .map((e) => e.targetId)
+    );
+    return Array.from(graph.nodes.values())
+      .filter((n) => n.type === 'code' && !documentedCode.has(n.id))
+      .map((n) => n.filePath ?? n.id);
+  }
+
+  traceChain(
+    graph: CrossReferenceGraph,
+    startId: string,
+    maxDepth: number = 3
+  ): string[][] {
+    const chains: string[][] = [];
+    const queue: Array<{ id: string; path: string[] }> = [
+      { id: startId, path: [startId] },
+    ];
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      if (current.path.length > maxDepth) continue;
+
+      const nextEdges = graph.edges.filter((e) => e.sourceId === current.id);
+      if (nextEdges.length === 0) {
+        chains.push([...current.path]);
+        continue;
+      }
+
+      for (const edge of nextEdges) {
+        if (!current.path.includes(edge.targetId)) {
+          queue.push({ id: edge.targetId, path: [...current.path, edge.targetId] });
+        }
+      }
+    }
+
+    return chains;
+  }
+}