@comfanion/usethis_search 0.1.4 → 0.2.0-dev.0

package/vectorizer/chunkers/code-chunker.ts

@@ -0,0 +1,325 @@
+/**
+ * Code Chunker — splits source code by functions, classes, and exports.
+ *
+ * Uses regex-based parsing (no AST dependency) to detect function/class
+ * boundaries. Falls back to line-based splitting for unstructured code.
+ */
+
+export interface CodeChunkConfig {
+  min_chunk_size: number
+  max_chunk_size: number
+  split_by_functions: boolean
+  include_function_signature: boolean
+}
+
+export const DEFAULT_CODE_CONFIG: CodeChunkConfig = {
+  min_chunk_size: 300,
+  max_chunk_size: 1500,
+  split_by_functions: true,
+  include_function_signature: true,
+}
+
+export interface CodeChunk {
+  content: string
+  function_name?: string
+  class_name?: string
+}
+
+// ── Block detection ─────────────────────────────────────────────────────────
+
+interface CodeBlock {
+  type: "function" | "class" | "method" | "other"
+  name: string
+  className?: string
+  startLine: number
+  endLine: number
+}
+
+/**
+ * Detect top-level function/class blocks via brace-counting.
+ * Works for JS/TS/Go/Rust/Java/C-family languages.
+ */
+function detectBlocks(lines: string[]): CodeBlock[] {
+  const blocks: CodeBlock[] = []
+
+  // Patterns for function/class declarations
+  const fnPatterns = [
+    // JS/TS: function name(, async function name(, export function
+    /(?:export\s+)?(?:async\s+)?function\s+(\w+)/,
+    // Arrow: const name = (…) =>  or  const name = async (
+    /(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>/,
+    // Method inside class: name( or async name(
+    /^\s+(?:async\s+)?(\w+)\s*\([^)]*\)\s*(?::\s*\w[^{]*)?\s*\{/,
+    // Go: func Name(
+    /^func\s+(?:\([^)]*\)\s+)?(\w+)\s*\(/,
+    // Rust: fn name(
+    /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)/,
+    // Python def
+    /^\s*(?:async\s+)?def\s+(\w+)\s*\(/,
+  ]
+
+  const classPatterns = [
+    // JS/TS/Java/C#: class Name
+    /(?:export\s+)?(?:abstract\s+)?class\s+(\w+)/,
+    // Rust: struct/enum/impl
+    /(?:pub\s+)?(?:struct|enum|impl)\s+(\w+)/,
+    // Python class
+    /^class\s+(\w+)/,
+  ]
+
+  let currentClass: string | undefined
+  let i = 0
+
+  while (i < lines.length) {
+    const line = lines[i]
+
+    // Check for class
+    let classMatch: RegExpMatchArray | null = null
+    for (const pat of classPatterns) {
+      classMatch = line.match(pat)
+      if (classMatch) break
+    }
+
+    if (classMatch) {
+      const name = classMatch[1]
+      const endLine = findBlockEnd(lines, i)
+      blocks.push({ type: "class", name, startLine: i, endLine })
+      currentClass = name
+
+      // Look for methods inside
+      for (let j = i + 1; j < endLine; j++) {
+        const methodLine = lines[j]
+        const methodMatch = methodLine.match(/^\s+(?:(?:public|private|protected|static|async|override)\s+)*(\w+)\s*\([^)]*\)\s*(?::\s*[^{]*)?\s*\{/)
+        if (methodMatch && methodMatch[1] !== "constructor" || methodMatch && methodMatch[1] === "constructor") {
+          const mEnd = findBlockEnd(lines, j)
+          blocks.push({
+            type: "method",
+            name: methodMatch[1],
+            className: name,
+            startLine: j,
+            endLine: mEnd,
+          })
+          j = mEnd
+        }
+      }
+
+      i = endLine + 1
+      currentClass = undefined
+      continue
+    }
+
+    // Check for standalone function
+    let fnMatch: RegExpMatchArray | null = null
+    for (const pat of fnPatterns) {
+      fnMatch = line.match(pat)
+      if (fnMatch) break
+    }
+
+    if (fnMatch && !currentClass) {
+      const name = fnMatch[1]
+      const endLine = findBlockEnd(lines, i)
+      blocks.push({ type: "function", name, startLine: i, endLine })
+      i = endLine + 1
+      continue
+    }
+
+    i++
+  }
+
+  return blocks
+}
+
+/** Find end of brace-delimited block starting at `startLine`. */
+function findBlockEnd(lines: string[], startLine: number): number {
+  let braceCount = 0
+  let started = false
+
+  // For Python-style (indent-based), use indent detection
+  const firstLine = lines[startLine]
+  const isPythonStyle = firstLine.match(/:\s*$/) && !firstLine.includes("{")
+
+  if (isPythonStyle) {
+    return findPythonBlockEnd(lines, startLine)
+  }
+
+  for (let i = startLine; i < lines.length; i++) {
+    const line = lines[i]
+    for (const ch of line) {
+      if (ch === "{") { braceCount++; started = true }
+      if (ch === "}") { braceCount-- }
+    }
+    if (started && braceCount <= 0) {
+      return i
+    }
+  }
+
+  return Math.min(startLine + 50, lines.length - 1)
+}
+
+/** Find end of indent-based block (Python). */
+function findPythonBlockEnd(lines: string[], startLine: number): number {
+  const baseIndent = lines[startLine].match(/^(\s*)/)?.[1].length ?? 0
+
+  for (let i = startLine + 1; i < lines.length; i++) {
+    const line = lines[i]
+    if (line.trim() === "") continue
+    const indent = line.match(/^(\s*)/)?.[1].length ?? 0
+    if (indent <= baseIndent) {
+      return i - 1
+    }
+  }
+
+  return lines.length - 1
+}
+
+// ── Fallback: line-based splitting ──────────────────────────────────────────
+
+function splitByLines(lines: string[], maxChars: number): CodeChunk[] {
+  const chunks: CodeChunk[] = []
+  let current: string[] = []
+  let currentLen = 0
+
+  for (const line of lines) {
+    if (currentLen + line.length + 1 > maxChars && current.length > 0) {
+      chunks.push({ content: current.join("\n") })
+      current = []
+      currentLen = 0
+    }
+    current.push(line)
+    currentLen += line.length + 1
+  }
+
+  if (current.length > 0) {
+    chunks.push({ content: current.join("\n") })
+  }
+
+  return chunks
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Chunk source code by functions/classes.
+ */
+export function chunkCode(
+  content: string,
+  config: CodeChunkConfig = DEFAULT_CODE_CONFIG,
+): CodeChunk[] {
+  const lines = content.split("\n")
+
+  if (!config.split_by_functions) {
+    return splitByLines(lines, config.max_chunk_size)
+  }
+
+  const blocks = detectBlocks(lines)
+
+  if (blocks.length === 0) {
+    // No recognizable blocks — fallback
+    return splitByLines(lines, config.max_chunk_size)
+  }
+
+  const chunks: CodeChunk[] = []
+
+  // Collect "gaps" (code between blocks) and blocks themselves
+  let lastEnd = -1
+
+  for (const block of blocks) {
+    // If there is class-level block, skip individual method-level duplicate
+    if (block.type === "method") continue
+
+    // Gap before this block
+    if (block.startLine > lastEnd + 1) {
+      const gapContent = lines.slice(lastEnd + 1, block.startLine).join("\n").trim()
+      if (gapContent.length >= config.min_chunk_size) {
+        chunks.push({ content: gapContent })
+      } else if (gapContent.length > 0 && chunks.length > 0) {
+        // Merge small gap with previous chunk
+        chunks[chunks.length - 1].content += "\n\n" + gapContent
+      } else if (gapContent.length > 0) {
+        chunks.push({ content: gapContent })
+      }
+    }
+
+    const blockContent = lines.slice(block.startLine, block.endLine + 1).join("\n")
+
+    if (blockContent.length > config.max_chunk_size && block.type === "class") {
+      // Split class into methods
+      const methods = blocks.filter(
+        (b) => b.type === "method" && b.className === block.name,
+      )
+
+      if (methods.length > 0) {
+        let classLastEnd = block.startLine
+
+        for (const method of methods) {
+          // Class preamble / gap before method
+          if (method.startLine > classLastEnd + 1) {
+            const gap = lines.slice(classLastEnd + 1, method.startLine).join("\n").trim()
+            if (gap) {
+              chunks.push({
+                content: gap,
+                class_name: block.name,
+              })
+            }
+          }
+
+          chunks.push({
+            content: lines.slice(method.startLine, method.endLine + 1).join("\n"),
+            function_name: method.name,
+            class_name: block.name,
+          })
+          classLastEnd = method.endLine
+        }
+
+        // Class tail
+        if (classLastEnd < block.endLine) {
+          const tail = lines.slice(classLastEnd + 1, block.endLine + 1).join("\n").trim()
+          if (tail) {
+            chunks.push({ content: tail, class_name: block.name })
+          }
+        }
+      } else {
+        // No methods found — split by lines
+        const subChunks = splitByLines(
+          lines.slice(block.startLine, block.endLine + 1),
+          config.max_chunk_size,
+        )
+        for (const sc of subChunks) {
+          sc.class_name = block.name
+          chunks.push(sc)
+        }
+      }
+    } else {
+      chunks.push({
+        content: blockContent,
+        function_name: block.type === "function" ? block.name : undefined,
+        class_name: block.type === "class" ? block.name : block.className,
+      })
+    }
+
+    lastEnd = block.endLine
+  }
+
+  // Trailing code after last block
+  if (lastEnd < lines.length - 1) {
+    const trailing = lines.slice(lastEnd + 1).join("\n").trim()
+    if (trailing.length > 0) {
+      chunks.push({ content: trailing })
+    }
+  }
+
+  // Final: split any chunk still too large
+  const result: CodeChunk[] = []
+  for (const chunk of chunks) {
+    if (chunk.content.length > config.max_chunk_size) {
+      const parts = splitByLines(chunk.content.split("\n"), config.max_chunk_size)
+      for (const p of parts) {
+        result.push({ ...chunk, content: p.content })
+      }
+    } else {
+      result.push(chunk)
+    }
+  }
+
+  return result.filter((c) => c.content.trim().length > 0)
+}

package/vectorizer/chunkers/markdown-chunker.ts

@@ -0,0 +1,177 @@
+/**
+ * Markdown Chunker — splits Markdown by heading structure.
+ *
+ * Preserves heading hierarchy ("API > Auth > JWT") in metadata,
+ * merges small sections, and splits oversized ones.
+ */
+
+export interface MarkdownChunkConfig {
+  min_chunk_size: number   // merge sections smaller than this (chars)
+  max_chunk_size: number   // split sections larger than this (chars)
+  split_by_headings: boolean
+  preserve_heading_hierarchy: boolean
+}
+
+export const DEFAULT_MD_CONFIG: MarkdownChunkConfig = {
+  min_chunk_size: 200,
+  max_chunk_size: 2000,
+  split_by_headings: true,
+  preserve_heading_hierarchy: true,
+}
+
+export interface MarkdownChunk {
+  content: string
+  heading_context: string   // "H1 > H2 > H3"
+}
+
+// ── Internal types ──────────────────────────────────────────────────────────
+
+interface Section {
+  level: number       // 1-6 for headings, 0 for preamble
+  heading: string
+  body: string
+}
+
+// ── Parsing ─────────────────────────────────────────────────────────────────
+
+/** Parse Markdown into sections keyed by heading. */
+function parseSections(content: string): Section[] {
+  const lines = content.split("\n")
+  const sections: Section[] = []
+  let currentSection: Section = { level: 0, heading: "", body: "" }
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,6})\s+(.+)$/)
+    if (headingMatch) {
+      // Push previous section
+      if (currentSection.body.trim() || currentSection.heading) {
+        sections.push(currentSection)
+      }
+      currentSection = {
+        level: headingMatch[1].length,
+        heading: headingMatch[2].trim(),
+        body: "",
+      }
+    } else {
+      currentSection.body += line + "\n"
+    }
+  }
+
+  // Push last section
+  if (currentSection.body.trim() || currentSection.heading) {
+    sections.push(currentSection)
+  }
+
+  return sections
+}
+
+/** Build heading hierarchy path for a section given the heading stack. */
+function buildHeadingContext(stack: { level: number; heading: string }[]): string {
+  return stack.map((h) => h.heading).join(" > ")
+}
+
+// ── Splitting oversized sections ────────────────────────────────────────────
+
+function splitLargeText(text: string, maxSize: number): string[] {
+  if (text.length <= maxSize) return [text]
+
+  const chunks: string[] = []
+  const lines = text.split("\n")
+  let current: string[] = []
+  let currentLen = 0
+
+  for (const line of lines) {
+    if (currentLen + line.length + 1 > maxSize && current.length > 0) {
+      chunks.push(current.join("\n"))
+      current = []
+      currentLen = 0
+    }
+    current.push(line)
+    currentLen += line.length + 1
+  }
+
+  if (current.length > 0) {
+    chunks.push(current.join("\n"))
+  }
+
+  return chunks
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Chunk Markdown content into semantic sections.
+ */
+export function chunkMarkdown(
+  content: string,
+  config: MarkdownChunkConfig = DEFAULT_MD_CONFIG,
+): MarkdownChunk[] {
+  if (!config.split_by_headings) {
+    // Fallback: single chunk (caller can use fixed chunker)
+    return [{ content, heading_context: "" }]
+  }
+
+  const sections = parseSections(content)
+  const rawChunks: MarkdownChunk[] = []
+
+  // Heading stack for hierarchy tracking
+  const headingStack: { level: number; heading: string }[] = []
+
+  for (const section of sections) {
+    // Update heading stack
+    if (section.level > 0) {
+      // Pop headings at same or deeper level
+      while (
+        headingStack.length > 0 &&
+        headingStack[headingStack.length - 1].level >= section.level
+      ) {
+        headingStack.pop()
+      }
+      headingStack.push({ level: section.level, heading: section.heading })
+    }
+
+    const headingContext = config.preserve_heading_hierarchy
+      ? buildHeadingContext(headingStack)
+      : section.heading
+
+    const sectionText = section.heading
+      ? `${"#".repeat(section.level)} ${section.heading}\n${section.body}`
+      : section.body
+
+    rawChunks.push({ content: sectionText.trim(), heading_context: headingContext })
+  }
+
+  // Merge small sections with previous
+  const merged: MarkdownChunk[] = []
+  for (const chunk of rawChunks) {
+    if (
+      merged.length > 0 &&
+      chunk.content.length < config.min_chunk_size
+    ) {
+      const prev = merged[merged.length - 1]
+      prev.content += "\n\n" + chunk.content
+      // Keep the deepest heading context
+      if (chunk.heading_context) {
+        prev.heading_context = chunk.heading_context
+      }
+    } else {
+      merged.push({ ...chunk })
+    }
+  }
+
+  // Split oversized sections
+  const result: MarkdownChunk[] = []
+  for (const chunk of merged) {
+    if (chunk.content.length > config.max_chunk_size) {
+      const parts = splitLargeText(chunk.content, config.max_chunk_size)
+      for (const part of parts) {
+        result.push({ content: part, heading_context: chunk.heading_context })
+      }
+    } else {
+      result.push(chunk)
+    }
+  }
+
+  // Filter empties
+  return result.filter((c) => c.content.trim().length > 0)
+}

package/vectorizer/content-cleaner.ts

@@ -0,0 +1,136 @@
+/**
+ * Content Cleaner — removes noise from file content before chunking.
+ *
+ * Strips TOC blocks, breadcrumbs, repeated headers, auto-generated markers,
+ * and optionally imports/comments so the embedding model sees only signal.
+ */
+
+export interface CleaningConfig {
+  remove_toc: boolean
+  remove_frontmatter_metadata: boolean
+  remove_imports: boolean
+  remove_comments: boolean
+}
+
+export const DEFAULT_CLEANING_CONFIG: CleaningConfig = {
+  remove_toc: true,
+  remove_frontmatter_metadata: false,
+  remove_imports: false,
+  remove_comments: false,
+}
+
+// ── Markdown noise ──────────────────────────────────────────────────────────
+
+/** Remove YAML front-matter (---…---) from Markdown. */
+function stripFrontmatter(text: string): string {
+  return text.replace(/^---\n[\s\S]*?\n---\n?/, "")
+}
+
+/**
+ * Remove inline TOC blocks.
+ * Matches patterns like:
+ *   ## Table of Contents
+ *   - [Section](#section)
+ *   …blank line
+ */
+function stripToc(text: string): string {
+  // Pattern: heading containing "table of contents" or "contents" followed by link-list
+  return text.replace(
+    /^#{1,3}\s*(Table of Contents|Contents|TOC)\s*\n([\t ]*[-*]\s*\[.*?\]\(#.*?\)\s*\n?)+/gim,
+    "",
+  )
+}
+
+/** Remove HTML-style TOC markers like <!-- TOC --> … <!-- /TOC --> */
+function stripHtmlTocMarkers(text: string): string {
+  return text.replace(/<!--\s*TOC\s*-->[\s\S]*?<!--\s*\/TOC\s*-->\n?/gi, "")
+}
+
+/** Remove breadcrumb lines, e.g. `Home > Docs > API` at the top. */
+function stripBreadcrumbs(text: string): string {
+  // Matches lines that look like breadcrumbs (word > word > word) at start
+  return text.replace(/^(?:[\w\s]+>\s*){2,}[\w\s]+\n{1,2}/gm, "")
+}
+
+/** Remove auto-generated code markers like `// AUTO-GENERATED` blocks. */
+function stripAutoGenMarkers(text: string): string {
+  return text.replace(
+    /\/[/*]\s*(?:AUTO[- ]?GENERATED|DO NOT (?:EDIT|MODIFY)|GENERATED BY|This file (?:is|was) (?:auto-?)?generated)[^\n]*/gi,
+    "",
+  )
+}
+
+// ── Code noise ──────────────────────────────────────────────────────────────
+
+/** Remove import/require statements (JS/TS/Python/Go). */
+function stripImports(text: string): string {
+  // JS/TS imports
+  let result = text.replace(/^import\s[\s\S]*?from\s+['"][^'"]+['"];?\s*$/gm, "")
+  result = result.replace(/^import\s+['"][^'"]+['"];?\s*$/gm, "")
+  // require
+  result = result.replace(/^(?:const|let|var)\s+.*?=\s*require\s*\(.*?\);?\s*$/gm, "")
+  // Python
+  result = result.replace(/^(?:from\s+\S+\s+)?import\s+.+$/gm, "")
+  // Go
+  result = result.replace(/^import\s*\(\s*\n(?:[\t ]*"[^"]*"\s*\n?)*\s*\)/gm, "")
+  result = result.replace(/^import\s+"[^"]*"\s*$/gm, "")
+  return result
+}
+
+/** Remove single-line and block comments (JS/TS style). */
+function stripComments(text: string): string {
+  // Block comments
+  let result = text.replace(/\/\*[\s\S]*?\*\//g, "")
+  // Single-line // comments (only full-line, not inline URLs etc.)
+  result = result.replace(/^\s*\/\/[^\n]*$/gm, "")
+  // Python/Ruby # comments (full line only)
+  result = result.replace(/^\s*#[^\n!]*$/gm, "")
+  return result
+}
+
+// ── Shared ──────────────────────────────────────────────────────────────────
+
+/** Collapse 3+ consecutive blank lines into 2. */
+function collapseBlankLines(text: string): string {
+  return text.replace(/\n{3,}/g, "\n\n")
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Clean file content according to the supplied config.
+ * @param content  Raw file content
+ * @param fileType 'docs' | 'code' | 'config'
+ * @param config   Cleaning options
+ */
+export function cleanContent(
+  content: string,
+  fileType: "docs" | "code" | "config",
+  config: CleaningConfig = DEFAULT_CLEANING_CONFIG,
+): string {
+  let result = content
+
+  if (fileType === "docs") {
+    if (config.remove_frontmatter_metadata) {
+      result = stripFrontmatter(result)
+    }
+    if (config.remove_toc) {
+      result = stripToc(result)
+      result = stripHtmlTocMarkers(result)
+    }
+    result = stripBreadcrumbs(result)
+  }
+
+  if (fileType === "code") {
+    result = stripAutoGenMarkers(result)
+    if (config.remove_imports) {
+      result = stripImports(result)
+    }
+    if (config.remove_comments) {
+      result = stripComments(result)
+    }
+  }
+
+  result = collapseBlankLines(result).trim()
+  return result
+}