@bahulam/code 0.1.1

package/src/context/prose-chunker.mjs

@@ -0,0 +1,255 @@
+/**
+ * Prose document chunker — JS port of the backend's Python chunker.
+ *
+ * Mirrors `codekepler-backend/app/agent/tools/shared/documents.py` so a
+ * file chunked here (CLI local path) and a file chunked server-side
+ * (chat upload or server `path` mode) produce byte-comparable chunks
+ * with the same page/chunk numbering. Callers can slice/reference
+ * chunks by (page_no, chunk_no) across surfaces.
+ *
+ * Constants come from Django's Phase 1 inline chunker
+ * (retail/chat_uploads/services.py) — the single source of truth for
+ * the whole platform.
+ *
+ * ─────────────────────────────────────────────────────────────────────
+ * Retriever interface — this module produces chunks; a retriever ranks
+ * them. Today the CLI has one retriever (BM25 via
+ * src/context/retriever.mjs). Desktop apps and richer offline setups
+ * will add an embedding-backed retriever. Both consume the same chunk
+ * shape produced here so the swap is transparent to callers:
+ *
+ *   Chunk (produced by this module):
+ *     { page: number|null, chunk_no: number, text: string, tokens: number }
+ *
+ *   Retriever (any implementation):
+ *     addSource(sourceId, chunks): void
+ *     search(query, { topK, sources? }): Array<{
+ *       sourceId, page, chunk_no, text, score
+ *     }>
+ *     removeSource(sourceId): void    // invalidate on file change
+ *
+ * The `read_attachment` tool talks to the retriever interface, never
+ * to a specific implementation. Adding embeddings later means writing
+ * an `EmbeddingRetriever` that satisfies this contract — no changes to
+ * the chunker or the tool.
+ * ─────────────────────────────────────────────────────────────────────
+ *
+ * Divergences from Python (documented):
+ *   - JS String indexing is UTF-16 code units, Python str is code points.
+ *     For BMP-only text (nearly all documents) the boundaries match
+ *     exactly. Non-BMP characters (emoji, some CJK) may fall a code unit
+ *     off vs Python; not a correctness issue for retrieval.
+ *   - TEXT_LIKE_MIMES is a JS-only superset of Python's TEXT_MIMES —
+ *     the CLI has historically supported json/yaml/html/log/rst, and
+ *     the chunker doesn't care about the surface syntax. Retrieval
+ *     quality on structured files (json/yaml) will be worse than on
+ *     prose; document that in the tool description, not here.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+// Byte-exact match with documents.py:29-31.
+export const CHUNK_TOKENS = 800;
+export const CHUNK_OVERLAP = 100;
+export const CHARS_PER_TOKEN = 4;
+
+// Strict server-parity set — same 5 mimes as documents.py.
+export const TEXT_MIMES = new Set([
+  'text/plain',
+  'text/markdown',
+  'text/csv',
+  'text/tab-separated-values',
+]);
+export const PDF_MIMES = new Set(['application/pdf']);
+export const DOCUMENT_MIMES = new Set([...TEXT_MIMES, ...PDF_MIMES]);
+
+// Wider set the CLI already supports at the `read_attachment` tool layer.
+// Chunking works fine on any UTF-8 text — retrieval quality on structured
+// formats is up to the retriever + query.
+export const TEXT_LIKE_MIMES = new Set([
+  ...TEXT_MIMES,
+  'application/json',
+  'application/x-yaml',
+  'application/toml',
+  'text/yaml',
+  'text/html',
+  'text/x-log',
+  'text/x-rst',
+  'text/x-restructuredtext',
+]);
+
+// Extension → mime map. Kept small and explicit — Python's
+// mimetypes.guess_type varies by OS registry; this table is stable.
+const EXT_TO_MIME = new Map([
+  ['.txt', 'text/plain'],
+  ['.log', 'text/x-log'],
+  ['.md', 'text/markdown'],
+  ['.markdown', 'text/markdown'],
+  ['.mdx', 'text/markdown'],       // MDX = markdown + JSX; treat as prose for retrieval
+  ['.csv', 'text/csv'],
+  ['.tsv', 'text/tab-separated-values'],
+  ['.json', 'application/json'],
+  ['.yaml', 'application/x-yaml'],
+  ['.yml', 'application/x-yaml'],
+  ['.toml', 'application/toml'],
+  ['.html', 'text/html'],
+  ['.htm', 'text/html'],
+  ['.rst', 'text/x-rst'],
+  ['.pdf', 'application/pdf'],
+]);
+
+/**
+ * Best-effort mime for a local file (extension-only, no magic-byte sniff).
+ * Files with no matching extension return 'application/octet-stream',
+ * consistent with Python's mimetypes.
+ */
+export function guessMime(filePath) {
+  const ext = path.extname(String(filePath || '')).toLowerCase();
+  return EXT_TO_MIME.get(ext) || 'application/octet-stream';
+}
+
+/**
+ * Chunk a text string with the 800/100 sliding window (character-based).
+ * Byte-comparable to Python's chunk_text() — see documents.py:40.
+ *
+ * @param {string} text
+ * @returns {Array<{chunk_no: number, text: string, tokens: number}>}
+ */
+export function chunkText(text) {
+  if (!text) return [];
+  const stepChars = (CHUNK_TOKENS - CHUNK_OVERLAP) * CHARS_PER_TOKEN;
+  const window = CHUNK_TOKENS * CHARS_PER_TOKEN;
+  const out = [];
+  let i = 0;
+  let chunkNo = 0;
+  const len = text.length;
+  while (i < len) {
+    const segment = text.substring(i, i + window);
+    const tokens = Math.max(1, Math.floor(segment.length / CHARS_PER_TOKEN));
+    out.push({ chunk_no: chunkNo, text: segment, tokens });
+    chunkNo += 1;
+    i += stepChars;
+  }
+  return out;
+}
+
+/**
+ * Extract per-page text from PDF bytes. Returns [{page, text}] with
+ * 1-indexed page numbers. Scanned/OCR-only pages come back with empty
+ * text — callers skip those (matches Python behavior).
+ *
+ * Uses the `pdf-parse` npm dep already in the CLI. Imported from
+ * `lib/pdf-parse.js` (not the default entry) to skip the debug-hook
+ * that opens a bundled test PDF at load time and fails in production.
+ */
+export async function extractPdfPages(buffer) {
+  const { default: pdfParse } = await import('pdf-parse/lib/pdf-parse.js');
+  const pageTexts = [];
+  try {
+    await pdfParse(buffer, {
+      // pdf-parse calls this per page in page order (pageIndex is 0-based).
+      // We accumulate into an array indexed by pageIndex to be defensive
+      // against any out-of-order rendering.
+      pagerender: async (pageData) => {
+        try {
+          const content = await pageData.getTextContent();
+          const text = (content.items || [])
+            .map(item => (typeof item.str === 'string' ? item.str : ''))
+            .join(' ');
+          const idx = typeof pageData.pageIndex === 'number' ? pageData.pageIndex : pageTexts.length;
+          pageTexts[idx] = text;
+          return text;
+        } catch {
+          return '';
+        }
+      },
+    });
+  } catch {
+    return [];
+  }
+  return pageTexts.map((text, i) => ({ page: i + 1, text: text || '' }));
+}
+
+/**
+ * Extract chunks from raw bytes. Returns the same shape as Python's
+ * extract_from_bytes(): [{page, chunk_no, text, tokens}].
+ *
+ * page is null for text mimes, 1-indexed for PDF pages.
+ * chunk_no is 0-indexed and monotonic across the whole document
+ * (matches Python `global_chunk` behavior at documents.py:109-115).
+ * Unsupported mimes return an empty array.
+ *
+ * @param {Buffer} buffer
+ * @param {string} mime
+ * @param {{textMimes?: Set<string>}} [opts] override which mimes are
+ *   treated as chunkable text. Defaults to TEXT_LIKE_MIMES (CLI's
+ *   permissive set). Pass TEXT_MIMES for strict server parity.
+ */
+export async function extractFromBytes(buffer, mime, opts = {}) {
+  const normalizedMime = String(mime || '').toLowerCase();
+  const textMimes = opts.textMimes || TEXT_LIKE_MIMES;
+
+  if (textMimes.has(normalizedMime)) {
+    const text = buffer.toString('utf8');
+    return chunkText(text).map(c => ({
+      page: null,
+      chunk_no: c.chunk_no,
+      text: c.text,
+      tokens: c.tokens,
+    }));
+  }
+
+  if (PDF_MIMES.has(normalizedMime)) {
+    const pages = await extractPdfPages(buffer);
+    const out = [];
+    let globalChunk = 0;
+    for (const { page, text } of pages) {
+      if (!text || !text.trim()) continue;
+      for (const c of chunkText(text)) {
+        out.push({ page, chunk_no: globalChunk, text: c.text, tokens: c.tokens });
+        globalChunk += 1;
+      }
+    }
+    return out;
+  }
+
+  return [];
+}
+
+/**
+ * Read a local file and chunk it. Returns { mime, chunks }; both empty
+ * on unresolvable path or unsupported mime — same contract as Python's
+ * extract_from_path(). Path resolution is the caller's responsibility
+ * (the CLI uses projectRegistry.resolvePath to enforce workspace bounds
+ * before calling here).
+ *
+ * @param {string} absPath  absolute, already-resolved file path
+ * @param {{textMimes?: Set<string>}} [opts]
+ * @returns {Promise<{mime: string, chunks: Array<object>}>}
+ */
+export async function extractFromPath(absPath, opts = {}) {
+  let stat;
+  try {
+    stat = fs.statSync(absPath);
+  } catch {
+    return { mime: '', chunks: [] };
+  }
+  if (!stat.isFile()) return { mime: '', chunks: [] };
+
+  const mime = guessMime(absPath);
+  const textMimes = opts.textMimes || TEXT_LIKE_MIMES;
+  if (!textMimes.has(mime) && !PDF_MIMES.has(mime)) {
+    return { mime, chunks: [] };
+  }
+
+  let buffer;
+  try {
+    buffer = fs.readFileSync(absPath);
+  } catch {
+    return { mime, chunks: [] };
+  }
+
+  const chunks = await extractFromBytes(buffer, mime, opts);
+  return { mime, chunks };
+}

package/src/context/retriever.mjs

@@ -0,0 +1,425 @@
+/**
+ * Context Retriever — T20: Unified context retrieval with BM25.
+ * Indexes project files and retrieves relevant chunks for LLM context.
+ */
+
+import { BM25Index } from './bm25.mjs';
+import { SymbolIndexer } from './symbol-indexer.mjs';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { indexDir as getIndexDir } from '../core/paths.mjs';
+
+const IGNORED_DIRS = new Set(['.git', 'node_modules', '.bahulam', '.kepler', '__pycache__', '.venv', 'venv', 'dist', 'build', '.next']);
+const CODE_EXTS = new Set(['.js', '.mjs', '.ts', '.tsx', '.py', '.go', '.rs', '.java', '.rb', '.php', '.c', '.cpp', '.h', '.css', '.html', '.json', '.yaml', '.yml', '.toml', '.md', '.sh']);
+const SYMBOL_EXTS = new Set(['.py', '.js', '.mjs', '.ts', '.tsx', '.jsx', '.go', '.rs']);
+const MAX_FILE_SIZE = 100_000; // 100KB
+const CHUNK_LINES = 50;
+const CHUNK_OVERLAP = 10;
+
+export class ContextRetriever {
+    constructor(projectDir = process.cwd()) {
+        this.projectDir = projectDir;
+        this.indexDir = getIndexDir(projectDir);
+        this.index = null;
+        this.symbolIndexer = null;
+        this.chunkTexts = new Map(); // id → original text content
+    }
+
+    /** Build or rebuild the search index (BM25 chunks + symbol index). */
+    async buildIndex() {
+        const files = this._scanFiles(this.projectDir);
+        const documents = [];
+
+        // Symbol indexer for AST-based search
+        this.symbolIndexer = new SymbolIndexer();
+        await this.symbolIndexer.init();
+
+        for (const filePath of files) {
+            try {
+                const content = fs.readFileSync(filePath, 'utf-8');
+                const relPath = path.relative(this.projectDir, filePath);
+
+                // BM25 chunks (existing behavior)
+                const chunks = this._chunkFile(content, relPath);
+                documents.push(...chunks);
+
+                // Symbol extraction for code files
+                const ext = path.extname(filePath).toLowerCase();
+                if (SYMBOL_EXTS.has(ext)) {
+                    await this.symbolIndexer.indexFile(relPath, content);
+                }
+            } catch { /* skip unreadable files */ }
+        }
+
+        this.index = new BM25Index();
+        this.index.buildIndex(documents);
+
+        // Store chunk texts for retrieval
+        this.chunkTexts = new Map();
+        for (const doc of documents) {
+            this.chunkTexts.set(doc.id, doc.text);
+        }
+
+        // Persist
+        if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+        fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+        fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        fs.writeFileSync(path.join(this.indexDir, 'symbols.json'), JSON.stringify(this.symbolIndexer.toJSON()));
+
+        return { fileCount: files.length, chunkCount: documents.length, symbolCount: this.symbolIndexer.symbolCount };
+    }
+
+    /**
+     * Incrementally update index for a single changed file.
+     * Re-chunks the file and replaces its entries in the BM25 index.
+     * ~5-50ms per file — safe to call after every edit/write.
+     */
+    updateFile(filePath) {
+        if (!this.index) {
+            if (!this.loadIndex()) return false;
+        }
+
+        const absPath = path.resolve(filePath);
+        const relPath = path.relative(this.projectDir, absPath);
+
+        // Remove old chunks for this file
+        const oldIds = new Set();
+        for (const doc of this.index.docs) {
+            if (doc.id === relPath || doc.id.startsWith(relPath + ':')) {
+                oldIds.add(doc.id);
+            }
+        }
+
+        // Collect remaining documents (excluding old chunks for this file)
+        const remainingDocs = [];
+        for (const doc of this.index.docs) {
+            if (!oldIds.has(doc.id)) {
+                // Reconstruct text from tf map for rebuild
+                remainingDocs.push({ id: doc.id, text: this.chunkTexts.get(doc.id) || '' });
+            }
+        }
+
+        // Remove old chunk texts
+        for (const id of oldIds) this.chunkTexts.delete(id);
+
+        // Re-chunk if file still exists (delete = just remove)
+        const newChunks = [];
+        if (fs.existsSync(absPath)) {
+            try {
+                const content = fs.readFileSync(absPath, 'utf-8');
+                newChunks.push(...this._chunkFile(content, relPath));
+            } catch { /* skip unreadable */ }
+        }
+
+        // Rebuild index from remaining + new chunks
+        // This is fast (~5-20ms) because BM25 build is O(n) on token count
+        const allDocs = [...remainingDocs, ...newChunks];
+        this.index = new BM25Index();
+        this.index.buildIndex(allDocs);
+
+        // Update chunk texts
+        for (const chunk of newChunks) {
+            this.chunkTexts.set(chunk.id, chunk.text);
+        }
+
+        // Persist updated index
+        try {
+            if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+            fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+            fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        } catch { /* best-effort persist */ }
+
+        return true;
+    }
+
+    /**
+     * Add already-chunked prose content (from prose-chunker.mjs) to the
+     * shared BM25 index — no re-read or re-chunking. Used by
+     * `read_attachment` to make docs discoverable by `search_code` /
+     * future `search_document` without a separate index.
+     *
+     * Chunk IDs are shaped `${sourceId}#c${chunk_no}` — the `#c`
+     * separator makes them distinguishable from code IDs (which use `:`)
+     * so `updateFile` for code files won't accidentally drop prose
+     * chunks and vice versa.
+     *
+     * Re-adding the same sourceId replaces its chunks (idempotent —
+     * safe to call on every `read_attachment`).
+     *
+     * @param {string} sourceId  stable id, usually a project-relative path
+     * @param {Array<{page: (number|null), chunk_no: number, text: string, tokens?: number}>} chunks
+     * @returns {number} chunks indexed
+     */
+    addProseChunks(sourceId, chunks) {
+        if (!Array.isArray(chunks) || chunks.length === 0) return 0;
+        if (!sourceId || typeof sourceId !== 'string') return 0;
+
+        if (!this.index) {
+            if (!this.loadIndex()) {
+                this.index = new BM25Index();
+                this.chunkTexts = new Map();
+            }
+        }
+
+        // Drop prior chunks for this source (idempotent).
+        const sourcePrefix = `${sourceId}#c`;
+        const oldIds = new Set();
+        for (const doc of this.index.docs) {
+            if (doc.id.startsWith(sourcePrefix)) oldIds.add(doc.id);
+        }
+        for (const id of oldIds) this.chunkTexts.delete(id);
+
+        // Collect surviving docs — reconstruct text from stored chunkTexts
+        // (BM25Index only stores tf maps + lengths, not raw text).
+        const remaining = [];
+        for (const doc of this.index.docs) {
+            if (!oldIds.has(doc.id)) {
+                remaining.push({ id: doc.id, text: this.chunkTexts.get(doc.id) || '' });
+            }
+        }
+
+        // Prep new prose docs — prefix indexed text with sourceId so BM25
+        // gets signal from the filename (mirrors how _chunkFile embeds
+        // relPath). Page tag included so page-scoped queries can hit it.
+        const newDocs = chunks.map(c => {
+            const id = `${sourceId}#c${c.chunk_no}`;
+            const pageTag = c.page != null ? ` page:${c.page}` : '';
+            const indexedText = `${sourceId}${pageTag}\n${c.text}`;
+            return { id, text: indexedText };
+        });
+
+        // Rebuild — BM25 needs IDF recomputed across all docs.
+        this.index = new BM25Index();
+        this.index.buildIndex([...remaining, ...newDocs]);
+        for (const doc of newDocs) {
+            this.chunkTexts.set(doc.id, doc.text);
+        }
+
+        // Persist. Best-effort — an unwritable indexDir shouldn't fail
+        // the tool call.
+        try {
+            if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+            fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+            fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        } catch { /* best-effort persist */ }
+
+        return newDocs.length;
+    }
+
+    /**
+     * Remove all chunks belonging to a source (either code path or
+     * prose sourceId). Called when a file is deleted so stale hits
+     * don't linger in the index.
+     */
+    removeSource(sourceId) {
+        if (!this.index) {
+            if (!this.loadIndex()) return 0;
+        }
+        const oldIds = new Set();
+        for (const doc of this.index.docs) {
+            // Code IDs: exact match or `sourceId:` prefix (line/AST chunks).
+            // Prose IDs: `sourceId#c` prefix.
+            if (
+                doc.id === sourceId
+                || doc.id.startsWith(`${sourceId}:`)
+                || doc.id.startsWith(`${sourceId}#c`)
+            ) {
+                oldIds.add(doc.id);
+            }
+        }
+        if (oldIds.size === 0) return 0;
+        for (const id of oldIds) this.chunkTexts.delete(id);
+
+        const remaining = [];
+        for (const doc of this.index.docs) {
+            if (!oldIds.has(doc.id)) {
+                remaining.push({ id: doc.id, text: this.chunkTexts.get(doc.id) || '' });
+            }
+        }
+
+        this.index = new BM25Index();
+        this.index.buildIndex(remaining);
+
+        try {
+            if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+            fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+            fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        } catch { /* best-effort */ }
+
+        return oldIds.size;
+    }
+
+    /** Load persisted index. */
+    loadIndex() {
+        const indexPath = path.join(this.indexDir, 'bm25.json');
+        const chunksPath = path.join(this.indexDir, 'chunks.json');
+        const symbolsPath = path.join(this.indexDir, 'symbols.json');
+        if (!fs.existsSync(indexPath)) return false;
+        try {
+            const data = JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
+            this.index = BM25Index.fromJSON(data);
+
+            if (fs.existsSync(chunksPath)) {
+                const chunks = JSON.parse(fs.readFileSync(chunksPath, 'utf-8'));
+                this.chunkTexts = new Map(Object.entries(chunks));
+            }
+
+            if (fs.existsSync(symbolsPath)) {
+                const symData = JSON.parse(fs.readFileSync(symbolsPath, 'utf-8'));
+                this.symbolIndexer = SymbolIndexer.fromJSON(symData);
+            }
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Search symbols (functions, classes, methods) by query.
+     * Returns structured results with file:line, signature, parent class.
+     */
+    searchSymbols(query, topK = 5) {
+        if (!this.symbolIndexer) return [];
+        return this.symbolIndexer.search(query, topK);
+    }
+
+    /**
+     * Format symbol search results for the agent.
+     */
+    formatSymbolResults(results) {
+        if (!this.symbolIndexer || !results.length) return '';
+        return this.symbolIndexer.formatResults(results);
+    }
+
+    /** Retrieve relevant context chunks for a query, with full text. */
+    retrieve(query, topK = 10) {
+        if (!this.index) {
+            if (!this.loadIndex()) return [];
+        }
+        const results = this.index.search(query, topK);
+        // Attach chunk text to results
+        return results.map(r => ({
+            ...r,
+            text: this.chunkTexts.get(r.id) || `[File: ${r.id}]`,
+        }));
+    }
+
+    /** Scan project files respecting .gitignore-like patterns. */
+    _scanFiles(dir, depth = 0) {
+        if (depth > 15) return [];
+        const results = [];
+        let entries;
+        try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
+
+        for (const entry of entries) {
+            if (entry.name.startsWith('.') && IGNORED_DIRS.has(entry.name)) continue;
+            if (IGNORED_DIRS.has(entry.name)) continue;
+
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                results.push(...this._scanFiles(fullPath, depth + 1));
+            } else if (entry.isFile()) {
+                const ext = path.extname(entry.name);
+                if (!CODE_EXTS.has(ext)) continue;
+                try {
+                    const stat = fs.statSync(fullPath);
+                    if (stat.size > MAX_FILE_SIZE) continue;
+                } catch { continue; }
+                results.push(fullPath);
+            }
+        }
+        return results;
+    }
+
+    /**
+     * Chunk a file by AST boundaries (functions/classes) with line-based fallback.
+     * AST-aware chunks contain complete functions/classes — not arbitrary 50-line blocks.
+     */
+    _chunkFile(content, relPath) {
+        const lines = content.split('\n');
+
+        // Small files: single chunk
+        if (lines.length <= CHUNK_LINES) {
+            return [{ id: relPath, text: `${relPath}\n${content}` }];
+        }
+
+        // Try AST-aware chunking: split at function/class boundaries
+        const boundaries = this._findASTBoundaries(lines);
+
+        if (boundaries.length > 1) {
+            // AST-aware: chunk at function/class boundaries
+            const chunks = [];
+            for (const { name, startLine, endLine } of boundaries) {
+                const chunk = lines.slice(startLine, endLine).join('\n');
+                const id = `${relPath}:${startLine + 1}:${name}`;
+                chunks.push({ id, text: `${relPath}:${startLine + 1} (${name})\n${chunk}` });
+            }
+            return chunks;
+        }
+
+        // Fallback: line-based chunking for non-code files
+        const chunks = [];
+        for (let i = 0; i < lines.length; i += (CHUNK_LINES - CHUNK_OVERLAP)) {
+            const chunk = lines.slice(i, i + CHUNK_LINES).join('\n');
+            chunks.push({ id: `${relPath}:${i + 1}`, text: `${relPath}:${i + 1}\n${chunk}` });
+        }
+        return chunks;
+    }
+
+    /**
+     * Find function/class boundaries using regex patterns.
+     * Returns array of { name, startLine, endLine }.
+     */
+    _findASTBoundaries(lines) {
+        const boundaries = [];
+        const patterns = [
+            // Python: def/class (indentation-based)
+            /^(?:async\s+)?def\s+(\w+)\s*\(/,
+            /^class\s+(\w+)/,
+            // JS/TS: function, class, export function
+            /^(?:export\s+)?(?:async\s+)?function\s+(\w+)/,
+            /^(?:export\s+)?class\s+(\w+)/,
+            // JS/TS: const fn = arrow
+            /^(?:export\s+)?(?:const|let)\s+(\w+)\s*=\s*(?:async\s+)?\(/,
+            // Go: func
+            /^func\s+(?:\([^)]*\)\s+)?(\w+)\s*\(/,
+            // Rust: fn, struct, impl
+            /^(?:pub\s+)?(?:async\s+)?fn\s+(\w+)/,
+            /^(?:pub\s+)?struct\s+(\w+)/,
+        ];
+
+        let currentStart = 0;
+        let currentName = '_top_level';
+
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i].trimStart();
+            for (const pattern of patterns) {
+                const match = line.match(pattern);
+                if (match) {
+                    // Close previous boundary
+                    if (i > currentStart) {
+                        boundaries.push({
+                            name: currentName,
+                            startLine: currentStart,
+                            endLine: i,
+                        });
+                    }
+                    currentStart = i;
+                    currentName = match[1] || match[2] || 'unknown';
+                    break;
+                }
+            }
+        }
+
+        // Close last boundary
+        if (lines.length > currentStart) {
+            boundaries.push({
+                name: currentName,
+                startLine: currentStart,
+                endLine: lines.length,
+            });
+        }
+
+        return boundaries;
+    }
+}