@comfanion/usethis_search 0.1.5 → 0.2.0-dev.0

package/vectorizer/chunkers/markdown-chunker.ts

@@ -0,0 +1,177 @@
+/**
+ * Markdown Chunker — splits Markdown by heading structure.
+ *
+ * Preserves heading hierarchy ("API > Auth > JWT") in metadata,
+ * merges small sections, and splits oversized ones.
+ */
+
+export interface MarkdownChunkConfig {
+  min_chunk_size: number   // merge sections smaller than this (chars)
+  max_chunk_size: number   // split sections larger than this (chars)
+  split_by_headings: boolean
+  preserve_heading_hierarchy: boolean
+}
+
+export const DEFAULT_MD_CONFIG: MarkdownChunkConfig = {
+  min_chunk_size: 200,
+  max_chunk_size: 2000,
+  split_by_headings: true,
+  preserve_heading_hierarchy: true,
+}
+
+export interface MarkdownChunk {
+  content: string
+  heading_context: string   // "H1 > H2 > H3"
+}
+
+// ── Internal types ──────────────────────────────────────────────────────────
+
+interface Section {
+  level: number       // 1-6 for headings, 0 for preamble
+  heading: string
+  body: string
+}
+
+// ── Parsing ─────────────────────────────────────────────────────────────────
+
+/** Parse Markdown into sections keyed by heading. */
+function parseSections(content: string): Section[] {
+  const lines = content.split("\n")
+  const sections: Section[] = []
+  let currentSection: Section = { level: 0, heading: "", body: "" }
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,6})\s+(.+)$/)
+    if (headingMatch) {
+      // Push previous section
+      if (currentSection.body.trim() || currentSection.heading) {
+        sections.push(currentSection)
+      }
+      currentSection = {
+        level: headingMatch[1].length,
+        heading: headingMatch[2].trim(),
+        body: "",
+      }
+    } else {
+      currentSection.body += line + "\n"
+    }
+  }
+
+  // Push last section
+  if (currentSection.body.trim() || currentSection.heading) {
+    sections.push(currentSection)
+  }
+
+  return sections
+}
+
+/** Build heading hierarchy path for a section given the heading stack. */
+function buildHeadingContext(stack: { level: number; heading: string }[]): string {
+  return stack.map((h) => h.heading).join(" > ")
+}
+
+// ── Splitting oversized sections ────────────────────────────────────────────
+
+function splitLargeText(text: string, maxSize: number): string[] {
+  if (text.length <= maxSize) return [text]
+
+  const chunks: string[] = []
+  const lines = text.split("\n")
+  let current: string[] = []
+  let currentLen = 0
+
+  for (const line of lines) {
+    if (currentLen + line.length + 1 > maxSize && current.length > 0) {
+      chunks.push(current.join("\n"))
+      current = []
+      currentLen = 0
+    }
+    current.push(line)
+    currentLen += line.length + 1
+  }
+
+  if (current.length > 0) {
+    chunks.push(current.join("\n"))
+  }
+
+  return chunks
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Chunk Markdown content into semantic sections.
+ */
+export function chunkMarkdown(
+  content: string,
+  config: MarkdownChunkConfig = DEFAULT_MD_CONFIG,
+): MarkdownChunk[] {
+  if (!config.split_by_headings) {
+    // Fallback: single chunk (caller can use fixed chunker)
+    return [{ content, heading_context: "" }]
+  }
+
+  const sections = parseSections(content)
+  const rawChunks: MarkdownChunk[] = []
+
+  // Heading stack for hierarchy tracking
+  const headingStack: { level: number; heading: string }[] = []
+
+  for (const section of sections) {
+    // Update heading stack
+    if (section.level > 0) {
+      // Pop headings at same or deeper level
+      while (
+        headingStack.length > 0 &&
+        headingStack[headingStack.length - 1].level >= section.level
+      ) {
+        headingStack.pop()
+      }
+      headingStack.push({ level: section.level, heading: section.heading })
+    }
+
+    const headingContext = config.preserve_heading_hierarchy
+      ? buildHeadingContext(headingStack)
+      : section.heading
+
+    const sectionText = section.heading
+      ? `${"#".repeat(section.level)} ${section.heading}\n${section.body}`
+      : section.body
+
+    rawChunks.push({ content: sectionText.trim(), heading_context: headingContext })
+  }
+
+  // Merge small sections with previous
+  const merged: MarkdownChunk[] = []
+  for (const chunk of rawChunks) {
+    if (
+      merged.length > 0 &&
+      chunk.content.length < config.min_chunk_size
+    ) {
+      const prev = merged[merged.length - 1]
+      prev.content += "\n\n" + chunk.content
+      // Keep the deepest heading context
+      if (chunk.heading_context) {
+        prev.heading_context = chunk.heading_context
+      }
+    } else {
+      merged.push({ ...chunk })
+    }
+  }
+
+  // Split oversized sections
+  const result: MarkdownChunk[] = []
+  for (const chunk of merged) {
+    if (chunk.content.length > config.max_chunk_size) {
+      const parts = splitLargeText(chunk.content, config.max_chunk_size)
+      for (const part of parts) {
+        result.push({ content: part, heading_context: chunk.heading_context })
+      }
+    } else {
+      result.push(chunk)
+    }
+  }
+
+  // Filter empties
+  return result.filter((c) => c.content.trim().length > 0)
+}

package/vectorizer/content-cleaner.ts

@@ -0,0 +1,136 @@
+/**
+ * Content Cleaner — removes noise from file content before chunking.
+ *
+ * Strips TOC blocks, breadcrumbs, repeated headers, auto-generated markers,
+ * and optionally imports/comments so the embedding model sees only signal.
+ */
+
+export interface CleaningConfig {
+  remove_toc: boolean
+  remove_frontmatter_metadata: boolean
+  remove_imports: boolean
+  remove_comments: boolean
+}
+
+export const DEFAULT_CLEANING_CONFIG: CleaningConfig = {
+  remove_toc: true,
+  remove_frontmatter_metadata: false,
+  remove_imports: false,
+  remove_comments: false,
+}
+
+// ── Markdown noise ──────────────────────────────────────────────────────────
+
+/** Remove YAML front-matter (---…---) from Markdown. */
+function stripFrontmatter(text: string): string {
+  return text.replace(/^---\n[\s\S]*?\n---\n?/, "")
+}
+
+/**
+ * Remove inline TOC blocks.
+ * Matches patterns like:
+ *   ## Table of Contents
+ *   - [Section](#section)
+ *   …blank line
+ */
+function stripToc(text: string): string {
+  // Pattern: heading containing "table of contents" or "contents" followed by link-list
+  return text.replace(
+    /^#{1,3}\s*(Table of Contents|Contents|TOC)\s*\n([\t ]*[-*]\s*\[.*?\]\(#.*?\)\s*\n?)+/gim,
+    "",
+  )
+}
+
+/** Remove HTML-style TOC markers like <!-- TOC --> … <!-- /TOC --> */
+function stripHtmlTocMarkers(text: string): string {
+  return text.replace(/<!--\s*TOC\s*-->[\s\S]*?<!--\s*\/TOC\s*-->\n?/gi, "")
+}
+
+/** Remove breadcrumb lines, e.g. `Home > Docs > API` at the top. */
+function stripBreadcrumbs(text: string): string {
+  // Matches lines that look like breadcrumbs (word > word > word) at start
+  return text.replace(/^(?:[\w\s]+>\s*){2,}[\w\s]+\n{1,2}/gm, "")
+}
+
+/** Remove auto-generated code markers like `// AUTO-GENERATED` blocks. */
+function stripAutoGenMarkers(text: string): string {
+  return text.replace(
+    /\/[/*]\s*(?:AUTO[- ]?GENERATED|DO NOT (?:EDIT|MODIFY)|GENERATED BY|This file (?:is|was) (?:auto-?)?generated)[^\n]*/gi,
+    "",
+  )
+}
+
+// ── Code noise ──────────────────────────────────────────────────────────────
+
+/** Remove import/require statements (JS/TS/Python/Go). */
+function stripImports(text: string): string {
+  // JS/TS imports
+  let result = text.replace(/^import\s[\s\S]*?from\s+['"][^'"]+['"];?\s*$/gm, "")
+  result = result.replace(/^import\s+['"][^'"]+['"];?\s*$/gm, "")
+  // require
+  result = result.replace(/^(?:const|let|var)\s+.*?=\s*require\s*\(.*?\);?\s*$/gm, "")
+  // Python
+  result = result.replace(/^(?:from\s+\S+\s+)?import\s+.+$/gm, "")
+  // Go
+  result = result.replace(/^import\s*\(\s*\n(?:[\t ]*"[^"]*"\s*\n?)*\s*\)/gm, "")
+  result = result.replace(/^import\s+"[^"]*"\s*$/gm, "")
+  return result
+}
+
+/** Remove single-line and block comments (JS/TS style). */
+function stripComments(text: string): string {
+  // Block comments
+  let result = text.replace(/\/\*[\s\S]*?\*\//g, "")
+  // Single-line // comments (only full-line, not inline URLs etc.)
+  result = result.replace(/^\s*\/\/[^\n]*$/gm, "")
+  // Python/Ruby # comments (full line only)
+  result = result.replace(/^\s*#[^\n!]*$/gm, "")
+  return result
+}
+
+// ── Shared ──────────────────────────────────────────────────────────────────
+
+/** Collapse 3+ consecutive blank lines into 2. */
+function collapseBlankLines(text: string): string {
+  return text.replace(/\n{3,}/g, "\n\n")
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Clean file content according to the supplied config.
+ * @param content  Raw file content
+ * @param fileType 'docs' | 'code' | 'config'
+ * @param config   Cleaning options
+ */
+export function cleanContent(
+  content: string,
+  fileType: "docs" | "code" | "config",
+  config: CleaningConfig = DEFAULT_CLEANING_CONFIG,
+): string {
+  let result = content
+
+  if (fileType === "docs") {
+    if (config.remove_frontmatter_metadata) {
+      result = stripFrontmatter(result)
+    }
+    if (config.remove_toc) {
+      result = stripToc(result)
+      result = stripHtmlTocMarkers(result)
+    }
+    result = stripBreadcrumbs(result)
+  }
+
+  if (fileType === "code") {
+    result = stripAutoGenMarkers(result)
+    if (config.remove_imports) {
+      result = stripImports(result)
+    }
+    if (config.remove_comments) {
+      result = stripComments(result)
+    }
+  }
+
+  result = collapseBlankLines(result).trim()
+  return result
+}

package/vectorizer/hybrid-search.ts

@@ -0,0 +1,97 @@
+/**
+ * Hybrid Search — merges vector similarity and BM25 keyword scores.
+ *
+ * Uses Reciprocal Rank Fusion (RRF) or weighted linear combination
+ * to merge results from two search backends.
+ */
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export interface HybridSearchConfig {
+  enabled: boolean
+  bm25_weight: number    // 0.0–1.0, vector_weight = 1 - bm25_weight
+}
+
+export const DEFAULT_HYBRID_CONFIG: HybridSearchConfig = {
+  enabled: false,
+  bm25_weight: 0.3,
+}
+
+export interface ScoredResult {
+  id: number              // index into the results array
+  vectorScore: number     // 0–1 (1 = best)
+  bm25Score: number       // raw BM25 score (unnormalized)
+  combinedScore: number
+}
+
+// ── Merge logic ─────────────────────────────────────────────────────────────
+
+/**
+ * Normalize BM25 scores to 0–1 range using min-max scaling.
+ */
+function normalizeBM25Scores(scores: Map<number, number>): Map<number, number> {
+  if (scores.size === 0) return scores
+
+  let min = Infinity
+  let max = -Infinity
+  for (const s of scores.values()) {
+    if (s < min) min = s
+    if (s > max) max = s
+  }
+
+  const range = max - min
+  if (range === 0) {
+    // All same score → normalize to 0.5
+    const result = new Map<number, number>()
+    for (const [id] of scores) result.set(id, 0.5)
+    return result
+  }
+
+  const result = new Map<number, number>()
+  for (const [id, score] of scores) {
+    result.set(id, (score - min) / range)
+  }
+  return result
+}
+
+/**
+ * Merge vector and BM25 results using weighted linear combination.
+ *
+ * @param vectorResults Map of chunkIndex → vectorScore (0–1, higher = better)
+ * @param bm25Results   Map of chunkIndex → raw BM25 score
+ * @param config        Hybrid search config (weights)
+ * @param limit         Max results to return
+ */
+export function mergeResults(
+  vectorResults: Map<number, number>,
+  bm25Results: Map<number, number>,
+  config: HybridSearchConfig = DEFAULT_HYBRID_CONFIG,
+  limit: number = 10,
+): ScoredResult[] {
+  const vectorWeight = 1 - config.bm25_weight
+  const bm25Weight = config.bm25_weight
+
+  const normalizedBM25 = normalizeBM25Scores(bm25Results)
+
+  // Collect all unique IDs
+  const allIds = new Set<number>()
+  for (const id of vectorResults.keys()) allIds.add(id)
+  for (const id of normalizedBM25.keys()) allIds.add(id)
+
+  const results: ScoredResult[] = []
+
+  for (const id of allIds) {
+    const vs = vectorResults.get(id) ?? 0
+    const bs = normalizedBM25.get(id) ?? 0
+
+    results.push({
+      id,
+      vectorScore: vs,
+      bm25Score: bm25Results.get(id) ?? 0,
+      combinedScore: vectorWeight * vs + bm25Weight * bs,
+    })
+  }
+
+  results.sort((a, b) => b.combinedScore - a.combinedScore)
+  return results.slice(0, limit)
+}