@comfanion/usethis_search 0.1.4 → 0.2.0-dev.0

package/tools/codeindex.ts

@@ -1,7 +1,8 @@
 /**
- * Code Index Status & Management Tool
+ * Code Index Status & Management Tool (v2)
  *
  * Uses bundled vectorizer. Index data is stored in `.opencode/vectors/<index>/`.
+ * v2: added "test" action for gold dataset testing, richer stats.
  */
 
 import { tool } from "@opencode-ai/plugin"
@@ -59,6 +60,7 @@ Actions:
 - "status" → Show index statistics
 - "list" → List all available indexes with stats
 - "reindex" → Re-index files using local vectorizer
+- "test" → Run gold dataset quality tests (if configured)
 
 Available indexes:
 - "code" - Source code files
@@ -66,7 +68,7 @@ Available indexes:
 - "config" - Configuration files`,
 
   args: {
-    action: tool.schema.enum(["status", "list", "reindex"]).describe("Action to perform"),
+    action: tool.schema.enum(["status", "list", "reindex", "test"]).describe("Action to perform"),
     index: tool.schema.string().optional().default("code").describe("Index name: code, docs, config"),
     dir: tool.schema.string().optional().describe("Directory to index (default: project root)"),
   },
@@ -87,7 +89,7 @@ Available indexes:
       } catch {}
 
       if (indexes.length === 0) {
-        output += `⚠️ No indexes created yet\n\nCreate indexes:\n\n\`\`\`\n`
+        output += `No indexes created yet\n\nCreate indexes:\n\n\`\`\`\n`
         output += `codeindex({ action: "reindex", index: "code" })\n`
         output += `codeindex({ action: "reindex", index: "docs", dir: "docs/" })\n`
         output += `\`\`\`\n`
@@ -95,31 +97,62 @@ Available indexes:
         output += `### Active Indexes\n\n`
         for (const idx of indexes) {
           try {
-            const hashesPath = path.join(vectorsDir, idx, "hashes.json")
-            const hashes = JSON.parse(await fs.readFile(hashesPath, "utf8"))
-            const fileCount = Object.keys(hashes).length
+            const indexer = await new CodebaseIndexer(projectRoot, idx).init()
+            const stats = await indexer.getStats()
+            await indexer.unloadModel()
             const desc = INDEX_DESCRIPTIONS[idx] || "Custom index"
-            output += `- ${idx} - ${desc} (files: ${fileCount})\n`
+            const features = stats.features
+              ? ` | chunking: ${stats.features.chunking}, hybrid: ${stats.features.hybrid ? "on" : "off"}`
+              : ""
+            output += `- **${idx}** - ${desc} (files: ${stats.fileCount}, chunks: ${stats.chunkCount}${features})\n`
           } catch {
             output += `- ${idx}\n`
           }
         }
       }
 
-      output += `\n### Usage\n\n\`\`\`\nsearch({ query: "your query", index: "code" })\n\`\`\``
+      output += `\n### Usage\n\n\`\`\`\nsearch({ query: "your query", index: "code" })\nsearch({ query: "your query", hybrid: true })  // v2: hybrid search\nsearch({ query: "your query", fileType: "code", language: "typescript" })  // v2: filters\n\`\`\``
       return output
     }
 
     if (args.action === "status") {
       const hashesFile = path.join(vectorsDir, indexName, "hashes.json")
       try {
-        const hashesContent = await fs.readFile(hashesFile, "utf8")
-        const hashes = JSON.parse(hashesContent)
-        const fileCount = Object.keys(hashes).length
-        const sampleFiles = Object.keys(hashes).slice(0, 5)
+        const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+        const stats = await indexer.getStats()
+        await indexer.unloadModel()
+
+        const sampleFiles = Object.keys(JSON.parse(await fs.readFile(hashesFile, "utf8"))).slice(0, 5)
         const desc = INDEX_DESCRIPTIONS[indexName] || "Custom index"
 
-        return `## Index Status: "${indexName}"\n\n**Description:** ${desc}\n**Files indexed:** ${fileCount}\n\n**Sample indexed files:**\n${sampleFiles.map((f) => `- ${f}`).join("\n")}${fileCount > 5 ? `\n- ... and ${fileCount - 5} more` : ""}`
+        let output = `## Index Status: "${indexName}"\n\n`
+        output += `**Description:** ${desc}\n`
+        output += `**Files indexed:** ${stats.fileCount}\n`
+        output += `**Total chunks:** ${stats.chunkCount}\n`
+        output += `**Model:** ${stats.model}\n`
+
+        if (stats.features) {
+          output += `\n**Features:**\n`
+          output += `- Chunking strategy: ${stats.features.chunking}\n`
+          output += `- Hybrid search: ${stats.features.hybrid ? "enabled" : "disabled"}\n`
+          output += `- Metrics: ${stats.features.metrics ? "enabled" : "disabled"}\n`
+          output += `- Query cache: ${stats.features.cache ? "enabled" : "disabled"}\n`
+        }
+
+        // Show metrics summary if available
+        try {
+          const metrics = await indexer.getMetrics()
+          if (metrics.total_queries > 0) {
+            output += `\n**Search Metrics:**\n`
+            output += `- Total queries: ${metrics.total_queries}\n`
+            output += `- Avg results/query: ${metrics.avg_results_per_query.toFixed(1)}\n`
+            output += `- Zero results rate: ${(metrics.zero_results_rate * 100).toFixed(1)}%\n`
+            output += `- Avg relevance: ${metrics.avg_relevance.toFixed(3)}\n`
+          }
+        } catch {}
+
+        output += `\n**Sample indexed files:**\n${sampleFiles.map((f) => `- ${f}`).join("\n")}${stats.fileCount > 5 ? `\n- ... and ${stats.fileCount - 5} more` : ""}`
+        return output
       } catch {
         return `## Index Status: "${indexName}"\n\nIndex "${indexName}" not created yet. Create it with: codeindex({ action: "reindex", index: "${indexName}" })`
       }
@@ -148,12 +181,98 @@ Available indexes:
         await indexer.unloadModel()
         const stats = await indexer.getStats()
 
-        return `## Re-indexing Complete ✅\n\n**Index:** ${indexName}\n**Directory:** ${args.dir || "(project root)"}\n**Files found:** ${files.length}\n**Files indexed:** ${indexed}\n**Files unchanged:** ${skipped}\n**Total chunks:** ${stats.chunkCount}`
+        let output = `## Re-indexing Complete\n\n`
+        output += `**Index:** ${indexName}\n`
+        output += `**Directory:** ${args.dir || "(project root)"}\n`
+        output += `**Files found:** ${files.length}\n`
+        output += `**Files indexed:** ${indexed}\n`
+        output += `**Files unchanged:** ${skipped}\n`
+        output += `**Total chunks:** ${stats.chunkCount}\n`
+        if (stats.features) {
+          output += `**Chunking:** ${stats.features.chunking}\n`
+        }
+        return output
+      } catch (error: any) {
+        return `Re-indexing failed: ${error.message || String(error)}`
+      }
+    }
+
+    if (args.action === "test") {
+      try {
+        const goldPath = path.join(projectRoot, ".opencode", "vectors", "gold-dataset.yaml")
+        let goldContent: string
+        try {
+          goldContent = await fs.readFile(goldPath, "utf8")
+        } catch {
+          return `## Gold Dataset Test\n\nNo gold dataset found at: ${goldPath}\n\nCreate one with test queries and expected results.\nSee docs/search-plugin-upgrade-plan.md for format.`
+        }
+
+        // Simple YAML parsing for test queries
+        const tests: { query: string; expected_files: string[]; min_relevance: number; description?: string }[] = []
+        const queryBlocks = goldContent.split(/\n\s+-\s+query:\s*/)
+        for (const block of queryBlocks.slice(1)) {
+          const queryMatch = block.match(/^["']?([^"'\n]+)["']?/)
+          const filesMatch = block.match(/expected_files:\s*\n((?:\s+-\s+.+\n?)+)/)
+          const relMatch = block.match(/min_relevance:\s*([\d.]+)/)
+          const descMatch = block.match(/description:\s*["']?([^"'\n]+)/)
+
+          if (queryMatch) {
+            const expectedFiles = filesMatch
+              ? filesMatch[1].split("\n").map(l => l.replace(/^\s+-\s+["']?/, "").replace(/["']$/, "").trim()).filter(Boolean)
+              : []
+            tests.push({
+              query: queryMatch[1].trim(),
+              expected_files: expectedFiles,
+              min_relevance: relMatch ? parseFloat(relMatch[1]) : 0.7,
+              description: descMatch ? descMatch[1].trim() : undefined,
+            })
+          }
+        }
+
+        if (tests.length === 0) {
+          return `## Gold Dataset Test\n\nNo test queries found in gold dataset.`
+        }
+
+        const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+        let passed = 0
+        let failed = 0
+        let output = `## Gold Dataset Test Results\n\n`
+
+        for (const t of tests) {
+          const results = await indexer.search(t.query, 10, false)
+          const foundFiles = results.map((r: any) => r.file)
+          const foundExpected = t.expected_files.filter(f => foundFiles.includes(f))
+          const topScore = results.length > 0 && results[0]._distance != null
+            ? 1 - results[0]._distance
+            : 0
+
+          const pass = foundExpected.length >= Math.ceil(t.expected_files.length * 0.5) && topScore >= t.min_relevance
+
+          if (pass) {
+            passed++
+            output += `**PASS** Query: "${t.query}"\n`
+          } else {
+            failed++
+            output += `**FAIL** Query: "${t.query}"\n`
+          }
+
+          output += `  Found: ${foundFiles.slice(0, 3).map((f: string) => `${f} (${(1 - (results.find((r: any) => r.file === f)?._distance ?? 1)).toFixed(2)})`).join(", ")}\n`
+          if (foundExpected.length < t.expected_files.length) {
+            const missing = t.expected_files.filter(f => !foundFiles.includes(f))
+            output += `  Missing: ${missing.join(", ")}\n`
+          }
+          output += `\n`
+        }
+
+        await indexer.unloadModel()
+
+        output += `---\n**Summary:** ${passed}/${tests.length} tests passed (${Math.round(passed / tests.length * 100)}%)\n`
+        return output
       } catch (error: any) {
-        return `❌ Re-indexing failed: ${error.message || String(error)}`
+        return `Gold dataset test failed: ${error.message || String(error)}`
       }
     }
 
-    return `Unknown action: ${args.action}. Use: status, list, or reindex`
+    return `Unknown action: ${args.action}. Use: status, list, reindex, or test`
   },
 })

package/tools/search.ts

@@ -1,7 +1,8 @@
 /**
- * Semantic Code Search Tool
+ * Semantic Code Search Tool (v2)
  *
  * Uses local embeddings + LanceDB vector store via bundled vectorizer.
+ * v2: hybrid search, metadata filtering, rich result metadata.
  * Index data is stored in `.opencode/vectors/<index>/`.
  */
 
@@ -33,6 +34,13 @@ Examples:
     searchAll: tool.schema.boolean().optional().default(false).describe("Search all indexes instead of just one"),
     freshen: tool.schema.boolean().optional().default(true).describe("Auto-update stale files before searching (default: true)"),
     includeArchived: tool.schema.boolean().optional().default(false).describe("Include archived files in results (default: false). Files are archived if in /archive/ folder or have 'archived: true' in frontmatter."),
+    // v2 params
+    hybrid: tool.schema.boolean().optional().describe("Enable hybrid search (vector + BM25 keyword matching). Improves exact keyword recall."),
+    fileType: tool.schema.string().optional().describe("Filter by file type: 'code', 'docs', or 'config'"),
+    language: tool.schema.string().optional().describe("Filter by language: 'typescript', 'python', 'markdown', etc."),
+    modifiedAfter: tool.schema.string().optional().describe("Filter: only files modified after this ISO date (e.g. '2024-01-01')"),
+    modifiedBefore: tool.schema.string().optional().describe("Filter: only files modified before this ISO date"),
+    tags: tool.schema.string().optional().describe("Filter by frontmatter tags (comma-separated, e.g. 'auth,security')"),
   },
 
   async execute(args) {
@@ -43,6 +51,15 @@ Examples:
       const limit = args.limit || 10
       const indexName = args.index || "code"
 
+      // Build search options from v2 params
+      const searchOptions: Record<string, any> = {}
+      if (args.hybrid != null) searchOptions.hybrid = args.hybrid
+      if (args.fileType) searchOptions.fileType = args.fileType
+      if (args.language) searchOptions.language = args.language
+      if (args.modifiedAfter) searchOptions.modifiedAfter = args.modifiedAfter
+      if (args.modifiedBefore) searchOptions.modifiedBefore = args.modifiedBefore
+      if (args.tags) searchOptions.tags = args.tags.split(",").map((t: string) => t.trim()).filter(Boolean)
+
       // Auto-freshen stale files before searching
       if (args.freshen !== false) {
         const tempIndexer = await new CodebaseIndexer(projectRoot, indexName).init()
@@ -56,7 +73,7 @@ Examples:
         await tempIndexer.unloadModel()
 
         if (indexes.length === 0) {
-          return `❌ No indexes found. Create one with: codeindex({ action: "reindex", index: "code" })`
+          return `No indexes found. Create one with: codeindex({ action: "reindex", index: "code" })`
         }
 
         for (const idx of indexes) {
@@ -64,42 +81,60 @@ Examples:
           if (args.freshen !== false) {
             await indexer.freshen()
           }
-          const results = await indexer.search(args.query, limit, args.includeArchived)
+          const results = await indexer.search(args.query, limit, args.includeArchived, searchOptions)
           allResults.push(...results.map((r: any) => ({ ...r, _index: idx })))
           await indexer.unloadModel()
         }
 
-        allResults.sort((a, b) => (a._distance || 0) - (b._distance || 0))
+        allResults.sort((a, b) => {
+          // Prefer combinedScore (hybrid), fall back to distance
+          const scoreA = a._combinedScore ?? (a._distance != null ? 1 - a._distance : 0)
+          const scoreB = b._combinedScore ?? (b._distance != null ? 1 - b._distance : 0)
+          return scoreB - scoreA
+        })
         allResults = allResults.slice(0, limit)
       } else {
         const hashesFile = path.join(projectRoot, ".opencode", "vectors", indexName, "hashes.json")
         try {
           await fs.access(hashesFile)
         } catch {
-          return `❌ Index "${indexName}" not found. Create it with: codeindex({ action: "reindex", index: "${indexName}" })`
+          return `Index "${indexName}" not found. Create it with: codeindex({ action: "reindex", index: "${indexName}" })`
         }
 
         const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
-        const results = await indexer.search(args.query, limit, args.includeArchived)
+        const results = await indexer.search(args.query, limit, args.includeArchived, searchOptions)
         allResults = results.map((r: any) => ({ ...r, _index: indexName }))
         await indexer.unloadModel()
       }
 
       if (allResults.length === 0) {
         const scope = args.searchAll ? "any index" : `index "${indexName}"`
-        return `No results found in ${scope} for: "${args.query}"\n\nTry:\n- Different keywords\n- Re-index with: codeindex({ action: "reindex", index: "${indexName}" })`
+        return `No results found in ${scope} for: "${args.query}"\n\nTry:\n- Different keywords\n- Enable hybrid search: search({ query: "...", hybrid: true })\n- Re-index with: codeindex({ action: "reindex", index: "${indexName}" })`
       }
 
       const scope = args.searchAll ? "all indexes" : `index "${indexName}"`
-      let output = `## Search Results for: "${args.query}" (${scope})\n\n`
+      const hybridLabel = args.hybrid ? " [hybrid]" : ""
+      let output = `## Search Results for: "${args.query}" (${scope}${hybridLabel})\n\n`
 
       for (let i = 0; i < allResults.length; i++) {
         const r = allResults[i]
-        const score = r._distance ? (1 - r._distance).toFixed(3) : "N/A"
+        const score = r._combinedScore != null
+          ? r._combinedScore.toFixed(3)
+          : r._distance != null
+            ? (1 - r._distance).toFixed(3)
+            : "N/A"
         const indexLabel = args.searchAll ? ` [${r._index}]` : ""
 
+        // v2: show rich metadata when available
+        const metaParts: string[] = []
+        if (r.language && r.language !== "unknown") metaParts.push(r.language)
+        if (r.heading_context) metaParts.push(`"${r.heading_context}"`)
+        if (r.function_name) metaParts.push(`fn: ${r.function_name}`)
+        if (r.class_name) metaParts.push(`class: ${r.class_name}`)
+        const metaLine = metaParts.length > 0 ? ` (${metaParts.join(", ")})` : ""
+
         output += `### ${i + 1}. ${r.file}${indexLabel}\n`
-        output += `**Relevance:** ${score}\n\n`
+        output += `**Relevance:** ${score}${metaLine}\n\n`
         output += "```\n"
         const content = r.content.length > 500 ? r.content.substring(0, 500) + "\n... (truncated)" : r.content
         output += content
@@ -109,7 +144,7 @@ Examples:
       output += `---\n*Found ${allResults.length} results. Use Read tool to see full files.*`
       return output
     } catch (error: any) {
-      return `❌ Search failed: ${error.message || String(error)}`
+      return `Search failed: ${error.message || String(error)}`
     }
   },
 })

package/vectorizer/bm25-index.ts

@@ -0,0 +1,155 @@
+/**
+ * BM25 Index — keyword-based search using Okapi BM25 scoring.
+ *
+ * Builds an inverted index from chunk content and scores queries
+ * against it. Designed to complement vector similarity search.
+ */
+
+// ── BM25 parameters ────────────────────────────────────────────────────────
+
+const K1 = 1.2   // term frequency saturation
+const B = 0.75   // length normalization
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+interface DocEntry {
+  id: number
+  termFreqs: Map<string, number>
+  length: number     // total tokens
+}
+
+export interface BM25Result {
+  id: number
+  score: number
+}
+
+// ── Tokenizer ───────────────────────────────────────────────────────────────
+
+const STOP_WORDS = new Set([
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "shall", "can", "need", "must",
+  "and", "or", "but", "not", "no", "nor",
+  "in", "on", "at", "to", "for", "of", "with", "by", "from", "as",
+  "into", "about", "between", "through", "during", "before", "after",
+  "this", "that", "these", "those", "it", "its",
+  "i", "you", "he", "she", "we", "they", "me", "him", "her", "us", "them",
+  "my", "your", "his", "our", "their",
+  "what", "which", "who", "whom", "where", "when", "how", "why",
+  "if", "then", "else", "so", "than", "too", "very",
+])
+
+/**
+ * Tokenize text into lowercase terms, filtering stop words and short tokens.
+ */
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9_\-]/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 1 && !STOP_WORDS.has(t))
+}
+
+// ── BM25 Index class ────────────────────────────────────────────────────────
+
+export class BM25Index {
+  private docs: DocEntry[] = []
+  private invertedIndex: Map<string, Set<number>> = new Map()
+  private avgDocLength: number = 0
+  private docCount: number = 0
+
+  /**
+   * Build index from a list of text documents.
+   * Each document is identified by its array index.
+   */
+  build(documents: string[]): void {
+    this.docs = []
+    this.invertedIndex = new Map()
+
+    let totalLength = 0
+
+    for (let i = 0; i < documents.length; i++) {
+      const tokens = tokenize(documents[i])
+      const termFreqs = new Map<string, number>()
+
+      for (const token of tokens) {
+        termFreqs.set(token, (termFreqs.get(token) || 0) + 1)
+
+        if (!this.invertedIndex.has(token)) {
+          this.invertedIndex.set(token, new Set())
+        }
+        this.invertedIndex.get(token)!.add(i)
+      }
+
+      this.docs.push({ id: i, termFreqs, length: tokens.length })
+      totalLength += tokens.length
+    }
+
+    this.docCount = documents.length
+    this.avgDocLength = this.docCount > 0 ? totalLength / this.docCount : 0
+  }
+
+  /**
+   * Score a query against indexed documents.
+   * Returns array sorted by descending score.
+   */
+  search(query: string, limit: number = 50): BM25Result[] {
+    const queryTerms = tokenize(query)
+    if (queryTerms.length === 0) return []
+
+    // Collect candidate docs (any doc containing at least one query term)
+    const candidateIds = new Set<number>()
+    for (const term of queryTerms) {
+      const postings = this.invertedIndex.get(term)
+      if (postings) {
+        for (const id of postings) candidateIds.add(id)
+      }
+    }
+
+    if (candidateIds.size === 0) return []
+
+    // Score each candidate
+    const results: BM25Result[] = []
+
+    for (const docId of candidateIds) {
+      const doc = this.docs[docId]
+      let score = 0
+
+      for (const term of queryTerms) {
+        const tf = doc.termFreqs.get(term) || 0
+        if (tf === 0) continue
+
+        const df = this.invertedIndex.get(term)?.size || 0
+        const idf = Math.log((this.docCount - df + 0.5) / (df + 0.5) + 1)
+        const tfNorm = (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * (doc.length / this.avgDocLength)))
+
+        score += idf * tfNorm
+      }
+
+      if (score > 0) {
+        results.push({ id: docId, score })
+      }
+    }
+
+    results.sort((a, b) => b.score - a.score)
+    return results.slice(0, limit)
+  }
+
+  /** Number of indexed documents. */
+  get size(): number {
+    return this.docCount
+  }
+
+  /** Number of unique terms. */
+  get vocabularySize(): number {
+    return this.invertedIndex.size
+  }
+
+  /** Release all memory held by the index. */
+  clear(): void {
+    this.docs = []
+    this.invertedIndex = new Map()
+    this.avgDocLength = 0
+    this.docCount = 0
+  }
+}

package/vectorizer/chunkers/chunker-factory.ts

@@ -0,0 +1,98 @@
+/**
+ * Chunker Factory — selects the appropriate chunker based on file type.
+ *
+ * Routes: markdown → markdown-chunker, code → code-chunker, else → fixed.
+ */
+
+import { chunkMarkdown, type MarkdownChunkConfig, DEFAULT_MD_CONFIG } from "./markdown-chunker"
+import { chunkCode, type CodeChunkConfig, DEFAULT_CODE_CONFIG } from "./code-chunker"
+import type { FileType } from "../metadata-extractor"
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export type ChunkingStrategy = "fixed" | "semantic" | "hybrid"
+
+export interface ChunkingConfig {
+  strategy: ChunkingStrategy
+  markdown: MarkdownChunkConfig
+  code: CodeChunkConfig
+  fixed: { max_chars: number }
+}
+
+export const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {
+  strategy: "semantic",
+  markdown: DEFAULT_MD_CONFIG,
+  code: DEFAULT_CODE_CONFIG,
+  fixed: { max_chars: 1500 },
+}
+
+/** Unified chunk output from any chunker. */
+export interface UnifiedChunk {
+  content: string
+  heading_context?: string
+  function_name?: string
+  class_name?: string
+}
+
+// ── Fixed chunker (legacy) ──────────────────────────────────────────────────
+
+function chunkFixed(content: string, maxChars: number): UnifiedChunk[] {
+  const chunks: UnifiedChunk[] = []
+  const lines = content.split("\n")
+  let current: string[] = []
+  let currentLen = 0
+
+  for (const line of lines) {
+    if (currentLen + line.length + 1 > maxChars && current.length > 0) {
+      chunks.push({ content: current.join("\n") })
+      current = []
+      currentLen = 0
+    }
+    current.push(line)
+    currentLen += line.length + 1
+  }
+
+  if (current.length > 0) {
+    chunks.push({ content: current.join("\n") })
+  }
+
+  return chunks
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Chunk content using the appropriate strategy for the given file type.
+ */
+export function chunkContent(
+  content: string,
+  fileType: FileType,
+  language: string,
+  config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG,
+): UnifiedChunk[] {
+  // If strategy is "fixed", always use fixed chunker
+  if (config.strategy === "fixed") {
+    return chunkFixed(content, config.fixed.max_chars)
+  }
+
+  // Semantic or hybrid: pick by file type
+  if (fileType === "docs" || language === "markdown") {
+    const mdChunks = chunkMarkdown(content, config.markdown)
+    return mdChunks.map((c) => ({
+      content: c.content,
+      heading_context: c.heading_context,
+    }))
+  }
+
+  if (fileType === "code") {
+    const codeChunks = chunkCode(content, config.code)
+    return codeChunks.map((c) => ({
+      content: c.content,
+      function_name: c.function_name,
+      class_name: c.class_name,
+    }))
+  }
+
+  // Config files or unknown — fixed
+  return chunkFixed(content, config.fixed.max_chars)
+}