@thoughtflow/codebase 0.2.0

package/src/libs/codebase/knowledge/markdown-extractor.ts

@@ -0,0 +1,270 @@
+import * as fs from "node:fs";
+
+export interface ExtractedDocument {
+  id: string;
+  filePath: string;
+  title: string;
+  docType: 'guideline' | 'architecture' | 'decision' | 'business' | 'technical' | 'audit' | 'unknown';
+  summary?: string;
+  sections: DocumentSection[];
+  frontmatter: Record<string, unknown>;
+  references: string[];
+  codeReferences: string[];
+  qualityScore: number;
+  isStale: boolean;
+}
+
+export interface DocumentSection {
+  heading: string;
+  level: number;
+  content: string;
+  startLine: number;
+}
+
+export class MarkdownExtractor {
+  extract(filePath: string, content?: string): ExtractedDocument {
+    const text = content ?? this.readFile(filePath);
+    const lines = text.split("\n");
+    const frontmatter = this.parseFrontmatter(lines);
+    const sections = this.parseSections(lines);
+    const docType = this.classifyDocType(filePath, frontmatter.data, text);
+    const references = this.extractReferences(text);
+    const codeRefs = this.extractCodeReferences(text);
+    const qualityScore = this.calculateQuality(frontmatter, sections, text, filePath);
+    const isStale = this.checkStaleness(frontmatter);
+
+    return {
+      id: this.slugify(filePath),
+      filePath,
+      title: (frontmatter.data.title as string) ?? this.extractTitle(lines),
+      docType,
+      sections,
+      frontmatter: frontmatter.data,
+      references,
+      codeReferences: codeRefs,
+      qualityScore,
+      isStale,
+    };
+  }
+
+  extractAll(filePaths: string[], rootPath: string): ExtractedDocument[] {
+    return filePaths
+      .map((fp) => {
+        const fullPath = rootPath + "/" + fp;
+        try {
+          return this.extract(fullPath);
+        } catch {
+          return null;
+        }
+      })
+      .filter((d): d is ExtractedDocument => d !== null);
+  }
+
+  private readFile(filePath: string): string {
+    return fs.readFileSync(filePath, "utf8");
+  }
+
+  private parseFrontmatter(lines: string[]): { data: Record<string, unknown>; endLine: number } {
+    const data: Record<string, unknown> = {};
+    let endLine = -1;
+
+    if (lines.length === 0 || lines[0].trim() !== "---") {
+      return { data, endLine };
+    }
+
+    for (let i = 1; i < lines.length; i++) {
+      if (lines[i].trim() === "---") {
+        endLine = i;
+        break;
+      }
+    }
+
+    if (endLine === -1) return { data, endLine };
+
+    for (let i = 1; i < endLine; i++) {
+      const line = lines[i].trim();
+      const colonIdx = line.indexOf(":");
+      if (colonIdx === -1) continue;
+      const key = line.slice(0, colonIdx).trim();
+      const value = line.slice(colonIdx + 1).trim();
+      if (value.startsWith("[") && value.endsWith("]")) {
+        data[key] = value.slice(1, -1).split(",").map((s) => s.trim().replace(/^["']|["']$/g, ""));
+      } else if (value === "true" || value === "false") {
+        data[key] = value === "true";
+      } else if (/^-?\d+(\.\d+)?$/.test(value)) {
+        data[key] = parseFloat(value);
+      } else {
+        data[key] = value.replace(/^["']|["']$/g, "");
+      }
+    }
+
+    return { data, endLine };
+  }
+
+  private parseSections(lines: string[]): DocumentSection[] {
+    const sections: DocumentSection[] = [];
+    let currentHeading = "";
+    let currentLevel = 0;
+    let currentLines: string[] = [];
+    let currentStart = 0;
+
+    for (let i = 0; i < lines.length; i++) {
+      const match = lines[i].match(/^(#{1,6})\s+(.+)/);
+      if (match) {
+        if (currentLines.length > 0) {
+          sections.push({
+            heading: currentHeading,
+            level: currentLevel,
+            content: currentLines.join("\n").trim(),
+            startLine: currentStart + 1,
+          });
+        }
+        currentHeading = match[2].trim();
+        currentLevel = match[1].length;
+        currentLines = [];
+        currentStart = i + 1;
+      } else {
+        currentLines.push(lines[i]);
+      }
+    }
+
+    if (currentLines.length > 0 || currentHeading) {
+      sections.push({
+        heading: currentHeading || "root",
+        level: currentLevel,
+        content: currentLines.join("\n").trim(),
+        startLine: currentStart + 1,
+      });
+    }
+
+    return sections;
+  }
+
+  private extractTitle(lines: string[]): string {
+    for (const line of lines) {
+      const match = line.match(/^#\s+(.+)/);
+      if (match) return match[1].trim();
+    }
+    return "untitled";
+  }
+
+  private classifyDocType(
+    filePath: string,
+    frontmatter: Record<string, unknown>,
+    content: string
+  ): ExtractedDocument['docType'] {
+    const fp = filePath.toLowerCase();
+    const txt = content.toLowerCase();
+
+    if (fp.includes("architecture") || fp.includes("arch/") || txt.includes("architecture decision")) return "architecture";
+    if (fp.includes("decision") || fp.includes("adr") || txt.includes("decision record")) return "decision";
+    if (fp.includes("business") || fp.includes("biznes") || fp.includes("bp-") || fp.includes("business_plan")) return "business";
+    if (fp.includes("api") || fp.includes("spec") || fp.includes("openapi") || txt.includes("api specification")) return "technical";
+    if (fp.includes("audit")) return "audit";
+    if (fp.includes("guideline") || fp.includes("guide") || fp.includes("standard") || fp.includes("convention") || fp.includes("instruction")) return "guideline";
+    if (fp.includes("readme") || fp.includes("contributing")) return "technical";
+
+    const fmType = frontmatter.type as string | undefined;
+    if (fmType) {
+      const t = fmType.toLowerCase();
+      if (["architecture", "decision", "business", "technical", "audit", "guideline"].includes(t)) {
+        return t as ExtractedDocument['docType'];
+      }
+    }
+
+    return "unknown";
+  }
+
+  private extractReferences(text: string): string[] {
+    const refs: string[] = [];
+    const linkPattern = /\[([^\]]+)\]\(([^)]+)\)/g;
+    let match: RegExpExecArray | null;
+    while ((match = linkPattern.exec(text)) !== null) {
+      const target = match[2];
+      if (target && !target.startsWith("http") && !target.startsWith("#")) {
+        refs.push(target);
+      }
+    }
+    return [...new Set(refs)];
+  }
+
+  private extractCodeReferences(text: string): string[] {
+    const refs: string[] = [];
+    const codeRefPattern = /`([^`]+\.(?:ts|tsx|js|jsx|py|go|rs|java))`/g;
+    let match: RegExpExecArray | null;
+    while ((match = codeRefPattern.exec(text)) !== null) {
+      refs.push(match[1]);
+    }
+
+    const commentRefPattern = /\/\/\s*(?:@feature|@ref|@see|PROJECT-MCP):\s*(\S+)/g;
+    while ((match = commentRefPattern.exec(text)) !== null) {
+      refs.push(match[1]);
+    }
+
+    return [...new Set(refs)];
+  }
+
+  private calculateQuality(
+    frontmatter: { data: Record<string, unknown>; endLine: number },
+    sections: DocumentSection[],
+    text: string,
+    filePath: string
+  ): number {
+    let score = 0;
+
+    if (frontmatter.endLine > -1) score += 0.1;
+    if (frontmatter.data.title) score += 0.1;
+    if (frontmatter.data.tags) score += 0.05;
+    if (frontmatter.data.date || frontmatter.data.updated) score += 0.05;
+
+    if (sections.length >= 3) score += 0.1;
+    if (sections.length >= 5) score += 0.05;
+
+    const refs = this.extractReferences(text);
+    const codeRefs = this.extractCodeReferences(text);
+    if (refs.length > 0) score += 0.1;
+    if (codeRefs.length > 0) score += 0.1;
+
+    const wordCount = text.split(/\s+/).length;
+    if (wordCount > 200) score += 0.1;
+    if (wordCount > 500) score += 0.05;
+
+    const autoGenPatterns = [
+      /generated by/i, /auto-generated/i, /do not edit/i,
+      /this file is automatically/i, /created with/i,
+    ];
+    if (autoGenPatterns.some((p) => p.test(text))) score -= 0.2;
+
+    const chatPatterns = [
+      /^> /m, /^user:/im, /^assistant:/im, /^system:/im,
+      /conversation history/i, /token count/i,
+    ];
+    if (chatPatterns.some((p) => p.test(text))) score -= 0.3;
+
+    if (wordCount < 50) score -= 0.2;
+
+    return Math.max(0, Math.min(1, score));
+  }
+
+  private checkStaleness(frontmatter: { data: Record<string, unknown>; endLine: number }): boolean {
+    const dateStr = (frontmatter.data.date ?? frontmatter.data.updated ?? frontmatter.data.lastModified) as string | undefined;
+    if (!dateStr) return false;
+
+    try {
+      const date = new Date(dateStr);
+      const now = new Date();
+      const monthsOld = (now.getTime() - date.getTime()) / (1000 * 60 * 60 * 24 * 30);
+      return monthsOld > 6;
+    } catch {
+      return false;
+    }
+  }
+
+  private slugify(filePath: string): string {
+    return filePath
+      .replace(/\.md$/, "")
+      .replace(/[^a-z0-9]+/gi, "-")
+      .replace(/^-+|-+$/g, "")
+      .toLowerCase();
+  }
+}

package/src/libs/codebase/knowledge/summarizer.ts

@@ -0,0 +1,85 @@
+import { z } from "zod";
+
+export interface SummarizerConfig {
+  maxTokensPerChunk?: number;
+  concurrency?: number;
+}
+
+export interface SummarizedChunk {
+  chunkId: string;
+  summary: string;
+}
+
+export class Summarizer {
+  private readonly maxTokens: number;
+  private readonly concurrency: number;
+
+  constructor(
+    private callLLM: (prompt: string, schema: z.ZodType) => Promise<Record<string, unknown>>,
+    config: SummarizerConfig = {}
+  ) {
+    this.maxTokens = config.maxTokensPerChunk ?? 200;
+    this.concurrency = config.concurrency ?? 3;
+  }
+
+  async summarizeChunks(
+    chunks: Array<{ id: string; text: string; type: string }>
+  ): Promise<Map<string, string>> {
+    const summaries = new Map<string, string>();
+
+    const schema = z.object({
+      summary: z.string().max(this.maxTokens),
+    });
+
+    for (let i = 0; i < chunks.length; i += this.concurrency) {
+      const batch = chunks.slice(i, i + this.concurrency);
+      const results = await Promise.all(
+        batch.map(async (chunk) => {
+          const context = chunk.type === "documentation"
+            ? "Summarize this documentation section in 1-2 sentences. Focus on the key information it conveys."
+            : "Summarize this code fragment in 1 sentence. What does it do? Be specific about its purpose.";
+
+          const prompt = `${context}\n\n\`\`\`\n${chunk.text.slice(0, 2000)}\n\`\`\``;
+
+          try {
+            const result = await this.callLLM(prompt, schema);
+            return { id: chunk.id, summary: (result.summary as string) ?? "" };
+          } catch {
+            return { id: chunk.id, summary: "" };
+          }
+        })
+      );
+
+      for (const result of results) {
+        if (result.summary) {
+          summaries.set(result.id, result.summary);
+        }
+      }
+    }
+
+    return summaries;
+  }
+
+  async summarizeDocument(
+    title: string,
+    sections: Array<{ heading: string; content: string }>
+  ): Promise<string> {
+    const schema = z.object({
+      summary: z.string().max(500),
+    });
+
+    const sectionList = sections
+      .slice(0, 10)
+      .map((s) => `## ${s.heading}\n${s.content.slice(0, 500)}`)
+      .join("\n\n");
+
+    const prompt = `Summarize this document titled "${title}" in 1 paragraph. Cover the main topics and purpose.\n\n${sectionList}`;
+
+    try {
+      const result = await this.callLLM(prompt, schema);
+      return (result.summary as string) ?? "";
+    } catch {
+      return "";
+    }
+  }
+}

package/src/libs/codebase/search/bm25.ts

@@ -0,0 +1,109 @@
+interface Bm25Document {
+  id: string;
+  tokens: string[];
+  text: string;
+}
+
+export class Bm25Index {
+  private docs: Bm25Document[] = [];
+  private docFreq: Map<string, number> = new Map();
+  private avgDocLength = 0;
+  private totalTokens = 0;
+  private readonly k1 = 1.5;
+  private readonly b = 0.75;
+
+  addDocument(id: string, text: string): void {
+    const tokens = this.tokenize(text);
+    const doc: Bm25Document = { id, tokens, text };
+    this.docs.push(doc);
+
+    const seenTokens = new Set<string>();
+    for (const token of tokens) {
+      if (!seenTokens.has(token)) {
+        this.docFreq.set(token, (this.docFreq.get(token) ?? 0) + 1);
+        seenTokens.add(token);
+      }
+    }
+
+    this.totalTokens += tokens.length;
+    this.avgDocLength = this.totalTokens / this.docs.length;
+  }
+
+  addDocuments(docs: Array<{ id: string; text: string }>): void {
+    for (const doc of docs) {
+      this.addDocument(doc.id, doc.text);
+    }
+  }
+
+  removeDocument(id: string): void {
+    const idx = this.docs.findIndex((d) => d.id === id);
+    if (idx === -1) return;
+
+    const doc = this.docs[idx];
+    const seenTokens = new Set(doc.tokens);
+
+    for (const token of seenTokens) {
+      const freq = (this.docFreq.get(token) ?? 1) - 1;
+      if (freq <= 0) {
+        this.docFreq.delete(token);
+      } else {
+        this.docFreq.set(token, freq);
+      }
+    }
+
+    this.totalTokens -= doc.tokens.length;
+    this.docs.splice(idx, 1);
+    this.avgDocLength = this.totalTokens / Math.max(1, this.docs.length);
+  }
+
+  search(query: string, limit = 10): Array<{ id: string; score: number }> {
+    const queryTokens = this.tokenize(query);
+    const results: Array<{ id: string; score: number }> = [];
+
+    for (const doc of this.docs) {
+      const score = this.scoreDocument(doc, queryTokens);
+      if (score > 0) {
+        results.push({ id: doc.id, score });
+      }
+    }
+
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, limit);
+  }
+
+  clear(): void {
+    this.docs = [];
+    this.docFreq.clear();
+    this.avgDocLength = 0;
+    this.totalTokens = 0;
+  }
+
+  get size(): number {
+    return this.docs.length;
+  }
+
+  private scoreDocument(doc: Bm25Document, queryTokens: string[]): number {
+    const N = this.docs.length;
+    const docLen = doc.tokens.length;
+
+    return queryTokens.reduce((sum, qt) => {
+      const tf = doc.tokens.filter((t) => t === qt).length;
+      if (tf === 0) return sum;
+
+      const df = this.docFreq.get(qt) ?? 1;
+      const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+      const numerator = tf * (this.k1 + 1);
+      const denominator = tf + this.k1 * (1 - this.b + this.b * (docLen / this.avgDocLength));
+
+      return sum + idf * (numerator / denominator);
+    }, 0);
+  }
+
+  private tokenize(text: string): string[] {
+    return text
+      .toLowerCase()
+      .replace(/[^a-z0-9_\s]/g, " ")
+      .split(/\s+/)
+      .filter((t) => t.length >= 2);
+  }
+}

package/src/libs/codebase/search/hybrid-search.ts

@@ -0,0 +1,109 @@
+import type { IVectorStore, IEmbeddingProvider, VectorSearchOptions, VectorSearchResult, VectorChunk } from "@thoughtflow/core";
+import { Bm25Index } from "./bm25";
+
+export interface HybridSearchOptions extends VectorSearchOptions {
+  rrfK?: number;
+  vectorWeight?: number;
+  keywordWeight?: number;
+}
+
+export class HybridSearch {
+  private bm25: Bm25Index = new Bm25Index();
+  private initialized = false;
+
+  constructor(
+    private vectorStore: IVectorStore,
+    private embeddingProvider: IEmbeddingProvider
+  ) {}
+
+  indexDocuments(docs: Array<{ id: string; text: string }>): void {
+    this.bm25.addDocuments(docs);
+    this.initialized = true;
+  }
+
+  removeDocument(id: string): void {
+    this.bm25.removeDocument(id);
+  }
+
+  async search(
+    query: string,
+    options?: HybridSearchOptions
+  ): Promise<VectorSearchResult[]> {
+    const rrfK = options?.rrfK ?? 60;
+    const vectorWeight = options?.vectorWeight ?? 0.5;
+    const keywordWeight = options?.keywordWeight ?? 0.5;
+
+    const queryVector = await this.embeddingProvider.embedQuery(query);
+
+    const [vectorResults, keywordResults] = await Promise.all([
+      this.vectorStore.search(queryVector, {
+        limit: options?.limit ?? 20,
+        filter: options?.filter,
+      }),
+      Promise.resolve(this.bm25.search(query, options?.limit ?? 20)),
+    ]);
+
+    const fusedScores = new Map<string, { vectorRank: number; keywordRank: number; item: VectorChunk }>();
+
+    for (let i = 0; i < vectorResults.length; i++) {
+      fusedScores.set(vectorResults[i].item.id, {
+        vectorRank: i + 1,
+        keywordRank: Infinity,
+        item: vectorResults[i].item,
+      });
+    }
+
+    const keywordIndex = new Map<number, string>();
+    for (let i = 0; i < keywordResults.length; i++) {
+      keywordIndex.set(i + 1, keywordResults[i].id);
+      const existing = fusedScores.get(keywordResults[i].id);
+      if (existing) {
+        existing.keywordRank = i + 1;
+        const score = this.fuseScore(rrfK, { vector: existing.vectorRank, keyword: i + 1 }, vectorWeight, keywordWeight);
+        keywordIndex.set(i + 1, keywordResults[i].id);
+      }
+    }
+
+    const fused: Array<{ id: string; score: number; item: VectorChunk }> = [];
+
+    for (const [id, data] of fusedScores) {
+      const score = this.fuseScore(
+        rrfK,
+        { vector: data.vectorRank, keyword: data.keywordRank },
+        vectorWeight,
+        keywordWeight
+      );
+      fused.push({ id, score, item: data.item });
+    }
+
+    fused.sort((a, b) => b.score - a.score);
+    const limit = options?.limit ?? 10;
+    const offset = options?.offset ?? 0;
+
+    return fused.slice(offset, offset + limit).map((r) => ({
+      item: r.item,
+      score: r.score,
+    }));
+  }
+
+  clear(): void {
+    this.bm25.clear();
+    this.initialized = false;
+  }
+
+  private fuseScore(
+    k: number,
+    ranks: { vector: number; keyword: number },
+    vectorWeight: number,
+    keywordWeight: number
+  ): number {
+    let score = 0;
+    if (ranks.vector < Infinity) {
+      score += vectorWeight * (1 / (k + ranks.vector));
+    }
+    if (ranks.keyword < Infinity) {
+      score += keywordWeight * (1 / (k + ranks.keyword));
+    }
+    return score;
+  }
+}