@dreb/semantic-search 1.18.0

package/dist/tree-sitter-chunker.d.ts

@@ -0,0 +1,25 @@
+/**
+ * AST-aware code chunking using tree-sitter (WASM).
+ *
+ * Parses source files into syntax trees and extracts meaningful code constructs
+ * (functions, classes, methods, structs, etc.) as individual chunks. Gaps between
+ * extracted nodes are captured as file-level chunks when substantial.
+ */
+import type { Chunk, TreeSitterLanguage } from "./types.js";
+/**
+ * Initialize the tree-sitter WASM runtime. Must be called before parsing.
+ * Safe to call multiple times — subsequent calls are no-ops.
+ */
+export declare function initTreeSitter(): Promise<void>;
+/**
+ * Parse a source file with tree-sitter and extract AST-aware chunks.
+ *
+ * Returns chunks for functions, classes, methods, and other language-specific
+ * constructs, plus file-level chunks for substantial gaps between them.
+ *
+ * @param content - Raw source code text
+ * @param filePath - Relative file path (stored in chunk metadata)
+ * @param language - Tree-sitter language identifier
+ */
+export declare function chunkWithTreeSitter(content: string, filePath: string, language: TreeSitterLanguage): Promise<Chunk[]>;
+//# sourceMappingURL=tree-sitter-chunker.d.ts.map

package/dist/tree-sitter-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree-sitter-chunker.d.ts","sourceRoot":"","sources":["../src/tree-sitter-chunker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,KAAK,EAAE,KAAK,EAAa,kBAAkB,EAAE,MAAM,YAAY,CAAC;AA8KvE;;;GAGG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAuBpD;AAqJD;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,kBAAkB,GAC1B,OAAO,CAAC,KAAK,EAAE,CAAC,CAwElB","sourcesContent":["/**\n * AST-aware code chunking using tree-sitter (WASM).\n *\n * Parses source files into syntax trees and extracts meaningful code constructs\n * (functions, classes, methods, structs, etc.) as individual chunks. Gaps between\n * extracted nodes are captured as file-level chunks when substantial.\n */\n\nimport { readFileSync } from \"fs\";\nimport { createRequire } from \"module\";\nimport type { Node as TSNode } from \"web-tree-sitter\";\nimport type { Chunk, ChunkKind, TreeSitterLanguage } from \"./types.js\";\n\n// Use createRequire for resolving WASM paths in ESM context\nconst require = createRequire(import.meta.url);\n\n// ============================================================================\n// Types\n// ============================================================================\n\n/** Describes which AST node types to extract for a language and how to get names. */\ninterface NodeExtractor {\n\t/** The tree-sitter node type string. */\n\ttype: string;\n\t/** The ChunkKind to assign to extracted chunks. */\n\tkind: ChunkKind;\n\t/** How to extract the symbol name from the node. */\n\tgetName: (node: TSNode) => string | null;\n}\n\n/** Intermediate representation of an extracted AST region. */\ninterface ExtractedRegion {\n\tname: string | null;\n\tkind: ChunkKind;\n\tstartLine: number; // 1-indexed\n\tendLine: number; // 1-indexed, inclusive\n\tcontent: string;\n}\n\n// ============================================================================\n// Lazy Imports\n// ============================================================================\n\n// web-tree-sitter types imported dynamically to avoid top-level await\ntype ParserClass = typeof import(\"web-tree-sitter\").Parser;\ntype LanguageClass = typeof import(\"web-tree-sitter\").Language;\n\nlet Parser: ParserClass | null = null;\nlet Language: LanguageClass | null = null;\n\nlet initPromise: Promise<void> | null = null;\nlet initialized = false;\n\n// ============================================================================\n// Language Cache\n// ============================================================================\n\nconst languageCache = new Map<TreeSitterLanguage, import(\"web-tree-sitter\").Language>();\n\n/** Grammar WASM paths keyed by language. */\nconst GRAMMAR_PATHS: Record<TreeSitterLanguage, string> = {\n\ttypescript: \"tree-sitter-typescript/tree-sitter-typescript.wasm\",\n\ttsx: \"tree-sitter-typescript/tree-sitter-tsx.wasm\",\n\tjavascript: \"tree-sitter-javascript/tree-sitter-javascript.wasm\",\n\tpython: \"tree-sitter-python/tree-sitter-python.wasm\",\n\tgo: \"tree-sitter-go/tree-sitter-go.wasm\",\n\trust: \"tree-sitter-rust/tree-sitter-rust.wasm\",\n\tjava: \"tree-sitter-java/tree-sitter-java.wasm\",\n\tc: \"tree-sitter-c/tree-sitter-c.wasm\",\n\tcpp: \"tree-sitter-cpp/tree-sitter-cpp.wasm\",\n};\n\n// ============================================================================\n// Name Extractors\n// ============================================================================\n\n/** Get name from a node's `name` field. */\nfunction nameField(node: TSNode): string | null {\n\treturn node.childForFieldName(\"name\")?.text ?? null;\n}\n\n/** Get name for an arrow function assigned to a variable. */\nfunction arrowFunctionName(node: TSNode): string | null {\n\tconst parent = node.parent;\n\tif (parent?.type === \"variable_declarator\") {\n\t\treturn parent.childForFieldName(\"name\")?.text ?? null;\n\t}\n\treturn null;\n}\n\n/** Get name for C function_definition: name is in the function_declarator child. */\nfunction cFunctionName(node: TSNode): string | null {\n\tconst declarator = node.childForFieldName(\"declarator\");\n\tif (!declarator) return null;\n\t// function_declarator has a `declarator` field for the actual name\n\tif (declarator.type === \"function_declarator\") {\n\t\treturn declarator.childForFieldName(\"declarator\")?.text ?? null;\n\t}\n\treturn declarator.text ?? null;\n}\n\n/** Get name from an export_statement's inner declaration. */\nfunction exportName(node: TSNode): string | null {\n\tconst decl = node.childForFieldName(\"declaration\");\n\tif (!decl) {\n\t\t// Named export like `export { foo }` — use the full text isn't useful,\n\t\t// just return null for anonymous exports\n\t\treturn null;\n\t}\n\treturn decl.childForFieldName(\"name\")?.text ?? null;\n}\n\n// ============================================================================\n// Per-Language Node Extractors\n// ============================================================================\n\nconst TS_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_declaration\", kind: \"function\", getName: nameField },\n\t{ type: \"method_definition\", kind: \"method\", getName: nameField },\n\t{ type: \"class_declaration\", kind: \"class\", getName: nameField },\n\t{ type: \"interface_declaration\", kind: \"interface\", getName: nameField },\n\t{ type: \"type_alias_declaration\", kind: \"type_alias\", getName: nameField },\n\t{ type: \"export_statement\", kind: \"export\", getName: exportName },\n\t{ type: \"arrow_function\", kind: \"function\", getName: arrowFunctionName },\n];\n\nconst JS_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_declaration\", kind: \"function\", getName: nameField },\n\t{ type: \"method_definition\", kind: \"method\", getName: nameField },\n\t{ type: \"class_declaration\", kind: \"class\", getName: nameField },\n\t{ type: \"export_statement\", kind: \"export\", getName: exportName },\n\t{ type: \"arrow_function\", kind: \"function\", getName: arrowFunctionName },\n];\n\nconst PYTHON_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_definition\", kind: \"function\", getName: nameField },\n\t{ type: \"class_definition\", kind: \"class\", getName: nameField },\n];\n\nconst GO_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_declaration\", kind: \"function\", getName: nameField },\n\t{ type: \"method_declaration\", kind: \"method\", getName: nameField },\n\t{ type: \"type_spec\", kind: \"struct\", getName: nameField },\n];\n\nconst RUST_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_item\", kind: \"function\", getName: nameField },\n\t{ type: \"impl_item\", kind: \"impl\", getName: (n) => n.childForFieldName(\"type\")?.text ?? null },\n\t{ type: \"struct_item\", kind: \"struct\", getName: nameField },\n\t{ type: \"enum_item\", kind: \"enum\", getName: nameField },\n\t{ type: \"trait_item\", kind: \"interface\", getName: nameField },\n];\n\nconst JAVA_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"class_declaration\", kind: \"class\", getName: nameField },\n\t{ type: \"method_declaration\", kind: \"method\", getName: nameField },\n\t{ type: \"interface_declaration\", kind: \"interface\", getName: nameField },\n];\n\nconst C_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_definition\", kind: \"function\", getName: cFunctionName },\n\t{ type: \"struct_specifier\", kind: \"struct\", getName: nameField },\n];\n\nconst CPP_EXTRACTORS: NodeExtractor[] = [\n\t...C_EXTRACTORS,\n\t{ type: \"class_specifier\", kind: \"class\", getName: nameField },\n];\n\nconst LANGUAGE_EXTRACTORS: Record<TreeSitterLanguage, NodeExtractor[]> = {\n\ttypescript: TS_EXTRACTORS,\n\ttsx: TS_EXTRACTORS,\n\tjavascript: JS_EXTRACTORS,\n\tpython: PYTHON_EXTRACTORS,\n\tgo: GO_EXTRACTORS,\n\trust: RUST_EXTRACTORS,\n\tjava: JAVA_EXTRACTORS,\n\tc: C_EXTRACTORS,\n\tcpp: CPP_EXTRACTORS,\n};\n\n// ============================================================================\n// Initialization\n// ============================================================================\n\n/**\n * Initialize the tree-sitter WASM runtime. Must be called before parsing.\n * Safe to call multiple times — subsequent calls are no-ops.\n */\nexport async function initTreeSitter(): Promise<void> {\n\tif (initialized) return;\n\tif (initPromise) return initPromise;\n\n\tinitPromise = (async () => {\n\t\ttry {\n\t\t\tconst mod = await import(\"web-tree-sitter\");\n\t\t\tParser = mod.Parser;\n\t\t\tLanguage = mod.Language;\n\n\t\t\tconst wasmPath = require.resolve(\"web-tree-sitter/web-tree-sitter.wasm\");\n\t\t\tconst wasmBuf = readFileSync(wasmPath);\n\t\t\tawait Parser.init({ locateFile: () => wasmPath, wasmBinary: wasmBuf });\n\t\t\tinitialized = true;\n\t\t} catch (err) {\n\t\t\t// Reset so subsequent calls can retry instead of returning\n\t\t\t// the same rejected promise forever\n\t\t\tinitPromise = null;\n\t\t\tthrow err;\n\t\t}\n\t})();\n\n\treturn initPromise;\n}\n\n// ============================================================================\n// Language Loading\n// ============================================================================\n\n/** Load and cache a tree-sitter language grammar. */\nasync function loadLanguage(lang: TreeSitterLanguage): Promise<import(\"web-tree-sitter\").Language> {\n\tconst cached = languageCache.get(lang);\n\tif (cached) return cached;\n\n\tif (!Language) {\n\t\tthrow new Error(\"tree-sitter not initialized — call initTreeSitter() first\");\n\t}\n\n\tconst grammarPath = require.resolve(GRAMMAR_PATHS[lang]);\n\tconst loaded = await Language.load(grammarPath);\n\tlanguageCache.set(lang, loaded);\n\treturn loaded;\n}\n\n// ============================================================================\n// AST Extraction\n// ============================================================================\n\n/**\n * Walk the tree and collect nodes matching the target types.\n * Returns regions sorted by start position, with nested nodes skipped\n * (only outermost matches are kept).\n */\nfunction extractRegions(rootNode: TSNode, extractors: NodeExtractor[], _sourceLines: string[]): ExtractedRegion[] {\n\t// Gather all target node types\n\tconst typeToExtractors = new Map<string, NodeExtractor>();\n\tfor (const ext of extractors) {\n\t\ttypeToExtractors.set(ext.type, ext);\n\t}\n\n\tconst targetTypes = extractors.map((e) => e.type);\n\tconst candidates = rootNode.descendantsOfType(targetTypes);\n\n\t// Convert to regions\n\tconst raw: ExtractedRegion[] = [];\n\tfor (const node of candidates) {\n\t\tconst ext = typeToExtractors.get(node.type);\n\t\tif (!ext) continue;\n\n\t\t// For struct_specifier in C, only extract if it has a body (field_declaration_list)\n\t\tif (node.type === \"struct_specifier\") {\n\t\t\tconst hasBody = node.children.some((c) => c.type === \"field_declaration_list\");\n\t\t\tif (!hasBody) continue;\n\t\t}\n\n\t\t// For arrow_function, only extract if parent is variable_declarator (named assignment)\n\t\tif (node.type === \"arrow_function\") {\n\t\t\tif (node.parent?.type !== \"variable_declarator\") continue;\n\t\t}\n\n\t\tconst startLine = node.startPosition.row + 1; // 0→1 indexed\n\t\tconst endLine = node.endPosition.row + 1;\n\n\t\traw.push({\n\t\t\tname: ext.getName(node),\n\t\t\tkind: ext.kind,\n\t\t\tstartLine,\n\t\t\tendLine,\n\t\t\tcontent: node.text,\n\t\t});\n\t}\n\n\t// Sort by start line, then by end line descending (larger ranges first)\n\traw.sort((a, b) => a.startLine - b.startLine || b.endLine - a.endLine);\n\n\t// Remove nested regions — keep only outermost\n\tconst regions: ExtractedRegion[] = [];\n\tlet lastEndLine = -1;\n\n\tfor (const region of raw) {\n\t\tif (region.startLine > lastEndLine) {\n\t\t\tregions.push(region);\n\t\t\tlastEndLine = region.endLine;\n\t\t}\n\t\t// else: this region is nested inside the previous one — skip\n\t}\n\n\treturn regions;\n}\n\n// ============================================================================\n// Gap Collection\n// ============================================================================\n\n/** Minimum number of non-blank lines for a gap to become its own chunk. */\nconst MIN_GAP_LINES = 3;\n\n/**\n * Create file-level chunks for substantial code between extracted regions.\n */\nfunction collectGaps(\n\tregions: ExtractedRegion[],\n\tsourceLines: string[],\n\tfilePath: string,\n\tfileType: TreeSitterLanguage,\n): Chunk[] {\n\tconst gaps: Chunk[] = [];\n\tlet cursor = 1; // 1-indexed current line\n\n\tfor (const region of regions) {\n\t\tif (region.startLine > cursor) {\n\t\t\tconst gapLines = sourceLines.slice(cursor - 1, region.startLine - 1);\n\t\t\tconst nonBlank = gapLines.filter((l) => l.trim().length > 0).length;\n\t\t\tif (nonBlank > MIN_GAP_LINES) {\n\t\t\t\tgaps.push({\n\t\t\t\t\tfilePath,\n\t\t\t\t\tstartLine: cursor,\n\t\t\t\t\tendLine: region.startLine - 1,\n\t\t\t\t\tkind: \"file\",\n\t\t\t\t\tname: null,\n\t\t\t\t\tcontent: gapLines.join(\"\\n\"),\n\t\t\t\t\tfileType,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\tcursor = region.endLine + 1;\n\t}\n\n\t// Trailing gap after last region\n\tif (cursor <= sourceLines.length) {\n\t\tconst gapLines = sourceLines.slice(cursor - 1);\n\t\tconst nonBlank = gapLines.filter((l) => l.trim().length > 0).length;\n\t\tif (nonBlank > MIN_GAP_LINES) {\n\t\t\tgaps.push({\n\t\t\t\tfilePath,\n\t\t\t\tstartLine: cursor,\n\t\t\t\tendLine: sourceLines.length,\n\t\t\t\tkind: \"file\",\n\t\t\t\tname: null,\n\t\t\t\tcontent: gapLines.join(\"\\n\"),\n\t\t\t\tfileType,\n\t\t\t});\n\t\t}\n\t}\n\n\treturn gaps;\n}\n\n// ============================================================================\n// Public API\n// ============================================================================\n\n/**\n * Parse a source file with tree-sitter and extract AST-aware chunks.\n *\n * Returns chunks for functions, classes, methods, and other language-specific\n * constructs, plus file-level chunks for substantial gaps between them.\n *\n * @param content - Raw source code text\n * @param filePath - Relative file path (stored in chunk metadata)\n * @param language - Tree-sitter language identifier\n */\nexport async function chunkWithTreeSitter(\n\tcontent: string,\n\tfilePath: string,\n\tlanguage: TreeSitterLanguage,\n): Promise<Chunk[]> {\n\tif (!initialized || !Parser) {\n\t\tawait initTreeSitter();\n\t}\n\n\t// After init, Parser is guaranteed to be set\n\tconst ParserCtor = Parser!;\n\tconst lang = await loadLanguage(language);\n\tconst parser = new ParserCtor();\n\tparser.setLanguage(lang);\n\n\tconst tree = parser.parse(content);\n\tif (!tree) {\n\t\t// Parse failed — free the parser WASM memory before returning\n\t\tparser.delete();\n\t\tconst lines = content.split(\"\\n\");\n\t\treturn [\n\t\t\t{\n\t\t\t\tfilePath,\n\t\t\t\tstartLine: 1,\n\t\t\t\tendLine: lines.length,\n\t\t\t\tkind: \"file\",\n\t\t\t\tname: null,\n\t\t\t\tcontent,\n\t\t\t\tfileType: language,\n\t\t\t},\n\t\t];\n\t}\n\n\ttry {\n\t\tconst sourceLines = content.split(\"\\n\");\n\t\tconst extractors = LANGUAGE_EXTRACTORS[language];\n\t\tconst regions = extractRegions(tree.rootNode, extractors, sourceLines);\n\n\t\t// Convert regions to Chunk objects\n\t\tconst chunks: Chunk[] = regions.map((r) => ({\n\t\t\tfilePath,\n\t\t\tstartLine: r.startLine,\n\t\t\tendLine: r.endLine,\n\t\t\tkind: r.kind,\n\t\t\tname: r.name,\n\t\t\tcontent: r.content,\n\t\t\tfileType: language,\n\t\t}));\n\n\t\t// Add gap chunks\n\t\tconst gaps = collectGaps(regions, sourceLines, filePath, language);\n\n\t\t// Merge and sort by start line\n\t\tconst all = [...chunks, ...gaps];\n\t\tall.sort((a, b) => a.startLine - b.startLine);\n\n\t\t// If no regions were extracted, return the whole file as one chunk\n\t\tif (chunks.length === 0) {\n\t\t\treturn [\n\t\t\t\t{\n\t\t\t\t\tfilePath,\n\t\t\t\t\tstartLine: 1,\n\t\t\t\t\tendLine: sourceLines.length,\n\t\t\t\t\tkind: \"file\",\n\t\t\t\t\tname: null,\n\t\t\t\t\tcontent,\n\t\t\t\t\tfileType: language,\n\t\t\t\t},\n\t\t\t];\n\t\t}\n\n\t\treturn all;\n\t} finally {\n\t\ttree.delete();\n\t\tparser.delete();\n\t}\n}\n"]}

package/dist/tree-sitter-chunker.js

@@ -0,0 +1,357 @@
+/**
+ * AST-aware code chunking using tree-sitter (WASM).
+ *
+ * Parses source files into syntax trees and extracts meaningful code constructs
+ * (functions, classes, methods, structs, etc.) as individual chunks. Gaps between
+ * extracted nodes are captured as file-level chunks when substantial.
+ */
+import { readFileSync } from "fs";
+import { createRequire } from "module";
+// Use createRequire for resolving WASM paths in ESM context
+const require = createRequire(import.meta.url);
+let Parser = null;
+let Language = null;
+let initPromise = null;
+let initialized = false;
+// ============================================================================
+// Language Cache
+// ============================================================================
+const languageCache = new Map();
+/** Grammar WASM paths keyed by language. */
+const GRAMMAR_PATHS = {
+    typescript: "tree-sitter-typescript/tree-sitter-typescript.wasm",
+    tsx: "tree-sitter-typescript/tree-sitter-tsx.wasm",
+    javascript: "tree-sitter-javascript/tree-sitter-javascript.wasm",
+    python: "tree-sitter-python/tree-sitter-python.wasm",
+    go: "tree-sitter-go/tree-sitter-go.wasm",
+    rust: "tree-sitter-rust/tree-sitter-rust.wasm",
+    java: "tree-sitter-java/tree-sitter-java.wasm",
+    c: "tree-sitter-c/tree-sitter-c.wasm",
+    cpp: "tree-sitter-cpp/tree-sitter-cpp.wasm",
+};
+// ============================================================================
+// Name Extractors
+// ============================================================================
+/** Get name from a node's `name` field. */
+function nameField(node) {
+    return node.childForFieldName("name")?.text ?? null;
+}
+/** Get name for an arrow function assigned to a variable. */
+function arrowFunctionName(node) {
+    const parent = node.parent;
+    if (parent?.type === "variable_declarator") {
+        return parent.childForFieldName("name")?.text ?? null;
+    }
+    return null;
+}
+/** Get name for C function_definition: name is in the function_declarator child. */
+function cFunctionName(node) {
+    const declarator = node.childForFieldName("declarator");
+    if (!declarator)
+        return null;
+    // function_declarator has a `declarator` field for the actual name
+    if (declarator.type === "function_declarator") {
+        return declarator.childForFieldName("declarator")?.text ?? null;
+    }
+    return declarator.text ?? null;
+}
+/** Get name from an export_statement's inner declaration. */
+function exportName(node) {
+    const decl = node.childForFieldName("declaration");
+    if (!decl) {
+        // Named export like `export { foo }` — use the full text isn't useful,
+        // just return null for anonymous exports
+        return null;
+    }
+    return decl.childForFieldName("name")?.text ?? null;
+}
+// ============================================================================
+// Per-Language Node Extractors
+// ============================================================================
+const TS_EXTRACTORS = [
+    { type: "function_declaration", kind: "function", getName: nameField },
+    { type: "method_definition", kind: "method", getName: nameField },
+    { type: "class_declaration", kind: "class", getName: nameField },
+    { type: "interface_declaration", kind: "interface", getName: nameField },
+    { type: "type_alias_declaration", kind: "type_alias", getName: nameField },
+    { type: "export_statement", kind: "export", getName: exportName },
+    { type: "arrow_function", kind: "function", getName: arrowFunctionName },
+];
+const JS_EXTRACTORS = [
+    { type: "function_declaration", kind: "function", getName: nameField },
+    { type: "method_definition", kind: "method", getName: nameField },
+    { type: "class_declaration", kind: "class", getName: nameField },
+    { type: "export_statement", kind: "export", getName: exportName },
+    { type: "arrow_function", kind: "function", getName: arrowFunctionName },
+];
+const PYTHON_EXTRACTORS = [
+    { type: "function_definition", kind: "function", getName: nameField },
+    { type: "class_definition", kind: "class", getName: nameField },
+];
+const GO_EXTRACTORS = [
+    { type: "function_declaration", kind: "function", getName: nameField },
+    { type: "method_declaration", kind: "method", getName: nameField },
+    { type: "type_spec", kind: "struct", getName: nameField },
+];
+const RUST_EXTRACTORS = [
+    { type: "function_item", kind: "function", getName: nameField },
+    { type: "impl_item", kind: "impl", getName: (n) => n.childForFieldName("type")?.text ?? null },
+    { type: "struct_item", kind: "struct", getName: nameField },
+    { type: "enum_item", kind: "enum", getName: nameField },
+    { type: "trait_item", kind: "interface", getName: nameField },
+];
+const JAVA_EXTRACTORS = [
+    { type: "class_declaration", kind: "class", getName: nameField },
+    { type: "method_declaration", kind: "method", getName: nameField },
+    { type: "interface_declaration", kind: "interface", getName: nameField },
+];
+const C_EXTRACTORS = [
+    { type: "function_definition", kind: "function", getName: cFunctionName },
+    { type: "struct_specifier", kind: "struct", getName: nameField },
+];
+const CPP_EXTRACTORS = [
+    ...C_EXTRACTORS,
+    { type: "class_specifier", kind: "class", getName: nameField },
+];
+const LANGUAGE_EXTRACTORS = {
+    typescript: TS_EXTRACTORS,
+    tsx: TS_EXTRACTORS,
+    javascript: JS_EXTRACTORS,
+    python: PYTHON_EXTRACTORS,
+    go: GO_EXTRACTORS,
+    rust: RUST_EXTRACTORS,
+    java: JAVA_EXTRACTORS,
+    c: C_EXTRACTORS,
+    cpp: CPP_EXTRACTORS,
+};
+// ============================================================================
+// Initialization
+// ============================================================================
+/**
+ * Initialize the tree-sitter WASM runtime. Must be called before parsing.
+ * Safe to call multiple times — subsequent calls are no-ops.
+ */
+export async function initTreeSitter() {
+    if (initialized)
+        return;
+    if (initPromise)
+        return initPromise;
+    initPromise = (async () => {
+        try {
+            const mod = await import("web-tree-sitter");
+            Parser = mod.Parser;
+            Language = mod.Language;
+            const wasmPath = require.resolve("web-tree-sitter/web-tree-sitter.wasm");
+            const wasmBuf = readFileSync(wasmPath);
+            await Parser.init({ locateFile: () => wasmPath, wasmBinary: wasmBuf });
+            initialized = true;
+        }
+        catch (err) {
+            // Reset so subsequent calls can retry instead of returning
+            // the same rejected promise forever
+            initPromise = null;
+            throw err;
+        }
+    })();
+    return initPromise;
+}
+// ============================================================================
+// Language Loading
+// ============================================================================
+/** Load and cache a tree-sitter language grammar. */
+async function loadLanguage(lang) {
+    const cached = languageCache.get(lang);
+    if (cached)
+        return cached;
+    if (!Language) {
+        throw new Error("tree-sitter not initialized — call initTreeSitter() first");
+    }
+    const grammarPath = require.resolve(GRAMMAR_PATHS[lang]);
+    const loaded = await Language.load(grammarPath);
+    languageCache.set(lang, loaded);
+    return loaded;
+}
+// ============================================================================
+// AST Extraction
+// ============================================================================
+/**
+ * Walk the tree and collect nodes matching the target types.
+ * Returns regions sorted by start position, with nested nodes skipped
+ * (only outermost matches are kept).
+ */
+function extractRegions(rootNode, extractors, _sourceLines) {
+    // Gather all target node types
+    const typeToExtractors = new Map();
+    for (const ext of extractors) {
+        typeToExtractors.set(ext.type, ext);
+    }
+    const targetTypes = extractors.map((e) => e.type);
+    const candidates = rootNode.descendantsOfType(targetTypes);
+    // Convert to regions
+    const raw = [];
+    for (const node of candidates) {
+        const ext = typeToExtractors.get(node.type);
+        if (!ext)
+            continue;
+        // For struct_specifier in C, only extract if it has a body (field_declaration_list)
+        if (node.type === "struct_specifier") {
+            const hasBody = node.children.some((c) => c.type === "field_declaration_list");
+            if (!hasBody)
+                continue;
+        }
+        // For arrow_function, only extract if parent is variable_declarator (named assignment)
+        if (node.type === "arrow_function") {
+            if (node.parent?.type !== "variable_declarator")
+                continue;
+        }
+        const startLine = node.startPosition.row + 1; // 0→1 indexed
+        const endLine = node.endPosition.row + 1;
+        raw.push({
+            name: ext.getName(node),
+            kind: ext.kind,
+            startLine,
+            endLine,
+            content: node.text,
+        });
+    }
+    // Sort by start line, then by end line descending (larger ranges first)
+    raw.sort((a, b) => a.startLine - b.startLine || b.endLine - a.endLine);
+    // Remove nested regions — keep only outermost
+    const regions = [];
+    let lastEndLine = -1;
+    for (const region of raw) {
+        if (region.startLine > lastEndLine) {
+            regions.push(region);
+            lastEndLine = region.endLine;
+        }
+        // else: this region is nested inside the previous one — skip
+    }
+    return regions;
+}
+// ============================================================================
+// Gap Collection
+// ============================================================================
+/** Minimum number of non-blank lines for a gap to become its own chunk. */
+const MIN_GAP_LINES = 3;
+/**
+ * Create file-level chunks for substantial code between extracted regions.
+ */
+function collectGaps(regions, sourceLines, filePath, fileType) {
+    const gaps = [];
+    let cursor = 1; // 1-indexed current line
+    for (const region of regions) {
+        if (region.startLine > cursor) {
+            const gapLines = sourceLines.slice(cursor - 1, region.startLine - 1);
+            const nonBlank = gapLines.filter((l) => l.trim().length > 0).length;
+            if (nonBlank > MIN_GAP_LINES) {
+                gaps.push({
+                    filePath,
+                    startLine: cursor,
+                    endLine: region.startLine - 1,
+                    kind: "file",
+                    name: null,
+                    content: gapLines.join("\n"),
+                    fileType,
+                });
+            }
+        }
+        cursor = region.endLine + 1;
+    }
+    // Trailing gap after last region
+    if (cursor <= sourceLines.length) {
+        const gapLines = sourceLines.slice(cursor - 1);
+        const nonBlank = gapLines.filter((l) => l.trim().length > 0).length;
+        if (nonBlank > MIN_GAP_LINES) {
+            gaps.push({
+                filePath,
+                startLine: cursor,
+                endLine: sourceLines.length,
+                kind: "file",
+                name: null,
+                content: gapLines.join("\n"),
+                fileType,
+            });
+        }
+    }
+    return gaps;
+}
+// ============================================================================
+// Public API
+// ============================================================================
+/**
+ * Parse a source file with tree-sitter and extract AST-aware chunks.
+ *
+ * Returns chunks for functions, classes, methods, and other language-specific
+ * constructs, plus file-level chunks for substantial gaps between them.
+ *
+ * @param content - Raw source code text
+ * @param filePath - Relative file path (stored in chunk metadata)
+ * @param language - Tree-sitter language identifier
+ */
+export async function chunkWithTreeSitter(content, filePath, language) {
+    if (!initialized || !Parser) {
+        await initTreeSitter();
+    }
+    // After init, Parser is guaranteed to be set
+    const ParserCtor = Parser;
+    const lang = await loadLanguage(language);
+    const parser = new ParserCtor();
+    parser.setLanguage(lang);
+    const tree = parser.parse(content);
+    if (!tree) {
+        // Parse failed — free the parser WASM memory before returning
+        parser.delete();
+        const lines = content.split("\n");
+        return [
+            {
+                filePath,
+                startLine: 1,
+                endLine: lines.length,
+                kind: "file",
+                name: null,
+                content,
+                fileType: language,
+            },
+        ];
+    }
+    try {
+        const sourceLines = content.split("\n");
+        const extractors = LANGUAGE_EXTRACTORS[language];
+        const regions = extractRegions(tree.rootNode, extractors, sourceLines);
+        // Convert regions to Chunk objects
+        const chunks = regions.map((r) => ({
+            filePath,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            kind: r.kind,
+            name: r.name,
+            content: r.content,
+            fileType: language,
+        }));
+        // Add gap chunks
+        const gaps = collectGaps(regions, sourceLines, filePath, language);
+        // Merge and sort by start line
+        const all = [...chunks, ...gaps];
+        all.sort((a, b) => a.startLine - b.startLine);
+        // If no regions were extracted, return the whole file as one chunk
+        if (chunks.length === 0) {
+            return [
+                {
+                    filePath,
+                    startLine: 1,
+                    endLine: sourceLines.length,
+                    kind: "file",
+                    name: null,
+                    content,
+                    fileType: language,
+                },
+            ];
+        }
+        return all;
+    }
+    finally {
+        tree.delete();
+        parser.delete();
+    }
+}
+//# sourceMappingURL=tree-sitter-chunker.js.map

package/dist/tree-sitter-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree-sitter-chunker.js","sourceRoot":"","sources":["../src/tree-sitter-chunker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAIvC,4DAA4D;AAC5D,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;AAiC/C,IAAI,MAAM,GAAuB,IAAI,CAAC;AACtC,IAAI,QAAQ,GAAyB,IAAI,CAAC;AAE1C,IAAI,WAAW,GAAyB,IAAI,CAAC;AAC7C,IAAI,WAAW,GAAG,KAAK,CAAC;AAExB,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E,MAAM,aAAa,GAAG,IAAI,GAAG,EAA0D,CAAC;AAExF,4CAA4C;AAC5C,MAAM,aAAa,GAAuC;IACzD,UAAU,EAAE,oDAAoD;IAChE,GAAG,EAAE,6CAA6C;IAClD,UAAU,EAAE,oDAAoD;IAChE,MAAM,EAAE,4CAA4C;IACpD,EAAE,EAAE,oCAAoC;IACxC,IAAI,EAAE,wCAAwC;IAC9C,IAAI,EAAE,wCAAwC;IAC9C,CAAC,EAAE,kCAAkC;IACrC,GAAG,EAAE,sCAAsC;CAC3C,CAAC;AAEF,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E,2CAA2C;AAC3C,SAAS,SAAS,CAAC,IAAY,EAAiB;IAC/C,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;AAAA,CACpD;AAED,6DAA6D;AAC7D,SAAS,iBAAiB,CAAC,IAAY,EAAiB;IACvD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3B,IAAI,MAAM,EAAE,IAAI,KAAK,qBAAqB,EAAE,CAAC;QAC5C,OAAO,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;IACvD,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,oFAAoF;AACpF,SAAS,aAAa,CAAC,IAAY,EAAiB;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACxD,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAC7B,mEAAmE;IACnE,IAAI,UAAU,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;QAC/C,OAAO,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;IACjE,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC;AAAA,CAC/B;AAED,6DAA6D;AAC7D,SAAS,UAAU,CAAC,IAAY,EAAiB;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,yEAAuE;QACvE,yCAAyC;QACzC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;AAAA,CACpD;AAED,+EAA+E;AAC/E,+BAA+B;AAC/B,+EAA+E;AAE/E,MAAM,aAAa,GAAoB;IACtC,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE;IACtE,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;IACjE,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE;IAChE,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE;IACxE,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE;IAC1E,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE;IACjE,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,iBAAiB,EAAE;CACxE,CAAC;AAEF,MAAM,aAAa,GAAoB;IACtC,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE;IACtE,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;IACjE,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE;IAChE,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE;IACjE,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,iBAAiB,EAAE;CACxE,CAAC;AAEF,MAAM,iBAAiB,GAAoB;IAC1C,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE;IACrE,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE;CAC/D,CAAC;AAEF,MAAM,aAAa,GAAoB;IACtC,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE;IACtE,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;IAClE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;CACzD,CAAC;AAEF,MAAM,eAAe,GAAoB;IACxC,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE;IAC/D,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE;IAC9F,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;IAC3D,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE;IACvD,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE;CAC7D,CAAC;AAEF,MAAM,eAAe,GAAoB;IACxC,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE;IAChE,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;IAClE,EAAE,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE;CACxE,CAAC;AAEF,MAAM,YAAY,GAAoB;IACrC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE;IACzE,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE;CAChE,CAAC;AAEF,MAAM,cAAc,GAAoB;IACvC,GAAG,YAAY;IACf,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE;CAC9D,CAAC;AAEF,MAAM,mBAAmB,GAAgD;IACxE,UAAU,EAAE,aAAa;IACzB,GAAG,EAAE,aAAa;IAClB,UAAU,EAAE,aAAa;IACzB,MAAM,EAAE,iBAAiB;IACzB,EAAE,EAAE,aAAa;IACjB,IAAI,EAAE,eAAe;IACrB,IAAI,EAAE,eAAe;IACrB,CAAC,EAAE,YAAY;IACf,GAAG,EAAE,cAAc;CACnB,CAAC;AAEF,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,GAAkB;IACrD,IAAI,WAAW;QAAE,OAAO;IACxB,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IAEpC,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAC5C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;YACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;YAExB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC;YACzE,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;YACvE,WAAW,GAAG,IAAI,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,2DAA2D;YAC3D,oCAAoC;YACpC,WAAW,GAAG,IAAI,CAAC;YACnB,MAAM,GAAG,CAAC;QACX,CAAC;IAAA,CACD,CAAC,EAAE,CAAC;IAEL,OAAO,WAAW,CAAC;AAAA,CACnB;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,qDAAqD;AACrD,KAAK,UAAU,YAAY,CAAC,IAAwB,EAA+C;IAClG,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6DAA2D,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAChD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAAA,CACd;AAED,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E;;;;GAIG;AACH,SAAS,cAAc,CAAC,QAAgB,EAAE,UAA2B,EAAE,YAAsB,EAAqB;IACjH,+BAA+B;IAC/B,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC1D,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAE3D,qBAAqB;IACrB,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,oFAAoF;QACpF,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,wBAAwB,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO;gBAAE,SAAS;QACxB,CAAC;QAED,uFAAuF;QACvF,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,KAAK,qBAAqB;gBAAE,SAAS;QAC3D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,gBAAc;QAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC;QAEzC,GAAG,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC;YACvB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS;YACT,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,IAAI;SAClB,CAAC,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAEvE,gDAA8C;IAC9C,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,WAAW,GAAG,CAAC,CAAC,CAAC;IAErB,KAAK,MAAM,MAAM,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,MAAM,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,CAAC;QACD,+DAA6D;IAC9D,CAAC;IAED,OAAO,OAAO,CAAC;AAAA,CACf;AAED,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E,2EAA2E;AAC3E,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB;;GAEG;AACH,SAAS,WAAW,CACnB,OAA0B,EAC1B,WAAqB,EACrB,QAAgB,EAChB,QAA4B,EAClB;IACV,MAAM,IAAI,GAAY,EAAE,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,yBAAyB;IAEzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,SAAS,GAAG,MAAM,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;YACrE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YACpE,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC;oBACT,QAAQ;oBACR,SAAS,EAAE,MAAM;oBACjB,OAAO,EAAE,MAAM,CAAC,SAAS,GAAG,CAAC;oBAC7B,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC5B,QAAQ;iBACR,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,iCAAiC;IACjC,IAAI,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC;gBACT,QAAQ;gBACR,SAAS,EAAE,MAAM;gBACjB,OAAO,EAAE,WAAW,CAAC,MAAM;gBAC3B,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI;gBACV,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,QAAQ;aACR,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAe,EACf,QAAgB,EAChB,QAA4B,EACT;IACnB,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,MAAM,cAAc,EAAE,CAAC;IACxB,CAAC;IAED,6CAA6C;IAC7C,MAAM,UAAU,GAAG,MAAO,CAAC;IAC3B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAChC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAEzB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,gEAA8D;QAC9D,MAAM,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO;YACN;gBACC,QAAQ;gBACR,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,KAAK,CAAC,MAAM;gBACrB,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI;gBACV,OAAO;gBACP,QAAQ,EAAE,QAAQ;aAClB;SACD,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;QAEvE,mCAAmC;QACnC,MAAM,MAAM,GAAY,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3C,QAAQ;YACR,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,QAAQ,EAAE,QAAQ;SAClB,CAAC,CAAC,CAAC;QAEJ,iBAAiB;QACjB,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEnE,+BAA+B;QAC/B,MAAM,GAAG,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;QACjC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QAE9C,mEAAmE;QACnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACN;oBACC,QAAQ;oBACR,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,WAAW,CAAC,MAAM;oBAC3B,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO;oBACP,QAAQ,EAAE,QAAQ;iBAClB;aACD,CAAC;QACH,CAAC;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACV,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,EAAE,CAAC;IACjB,CAAC;AAAA,CACD","sourcesContent":["/**\n * AST-aware code chunking using tree-sitter (WASM).\n *\n * Parses source files into syntax trees and extracts meaningful code constructs\n * (functions, classes, methods, structs, etc.) as individual chunks. Gaps between\n * extracted nodes are captured as file-level chunks when substantial.\n */\n\nimport { readFileSync } from \"fs\";\nimport { createRequire } from \"module\";\nimport type { Node as TSNode } from \"web-tree-sitter\";\nimport type { Chunk, ChunkKind, TreeSitterLanguage } from \"./types.js\";\n\n// Use createRequire for resolving WASM paths in ESM context\nconst require = createRequire(import.meta.url);\n\n// ============================================================================\n// Types\n// ============================================================================\n\n/** Describes which AST node types to extract for a language and how to get names. */\ninterface NodeExtractor {\n\t/** The tree-sitter node type string. */\n\ttype: string;\n\t/** The ChunkKind to assign to extracted chunks. */\n\tkind: ChunkKind;\n\t/** How to extract the symbol name from the node. */\n\tgetName: (node: TSNode) => string | null;\n}\n\n/** Intermediate representation of an extracted AST region. */\ninterface ExtractedRegion {\n\tname: string | null;\n\tkind: ChunkKind;\n\tstartLine: number; // 1-indexed\n\tendLine: number; // 1-indexed, inclusive\n\tcontent: string;\n}\n\n// ============================================================================\n// Lazy Imports\n// ============================================================================\n\n// web-tree-sitter types imported dynamically to avoid top-level await\ntype ParserClass = typeof import(\"web-tree-sitter\").Parser;\ntype LanguageClass = typeof import(\"web-tree-sitter\").Language;\n\nlet Parser: ParserClass | null = null;\nlet Language: LanguageClass | null = null;\n\nlet initPromise: Promise<void> | null = null;\nlet initialized = false;\n\n// ============================================================================\n// Language Cache\n// ============================================================================\n\nconst languageCache = new Map<TreeSitterLanguage, import(\"web-tree-sitter\").Language>();\n\n/** Grammar WASM paths keyed by language. */\nconst GRAMMAR_PATHS: Record<TreeSitterLanguage, string> = {\n\ttypescript: \"tree-sitter-typescript/tree-sitter-typescript.wasm\",\n\ttsx: \"tree-sitter-typescript/tree-sitter-tsx.wasm\",\n\tjavascript: \"tree-sitter-javascript/tree-sitter-javascript.wasm\",\n\tpython: \"tree-sitter-python/tree-sitter-python.wasm\",\n\tgo: \"tree-sitter-go/tree-sitter-go.wasm\",\n\trust: \"tree-sitter-rust/tree-sitter-rust.wasm\",\n\tjava: \"tree-sitter-java/tree-sitter-java.wasm\",\n\tc: \"tree-sitter-c/tree-sitter-c.wasm\",\n\tcpp: \"tree-sitter-cpp/tree-sitter-cpp.wasm\",\n};\n\n// ============================================================================\n// Name Extractors\n// ============================================================================\n\n/** Get name from a node's `name` field. */\nfunction nameField(node: TSNode): string | null {\n\treturn node.childForFieldName(\"name\")?.text ?? null;\n}\n\n/** Get name for an arrow function assigned to a variable. */\nfunction arrowFunctionName(node: TSNode): string | null {\n\tconst parent = node.parent;\n\tif (parent?.type === \"variable_declarator\") {\n\t\treturn parent.childForFieldName(\"name\")?.text ?? null;\n\t}\n\treturn null;\n}\n\n/** Get name for C function_definition: name is in the function_declarator child. */\nfunction cFunctionName(node: TSNode): string | null {\n\tconst declarator = node.childForFieldName(\"declarator\");\n\tif (!declarator) return null;\n\t// function_declarator has a `declarator` field for the actual name\n\tif (declarator.type === \"function_declarator\") {\n\t\treturn declarator.childForFieldName(\"declarator\")?.text ?? null;\n\t}\n\treturn declarator.text ?? null;\n}\n\n/** Get name from an export_statement's inner declaration. */\nfunction exportName(node: TSNode): string | null {\n\tconst decl = node.childForFieldName(\"declaration\");\n\tif (!decl) {\n\t\t// Named export like `export { foo }` — use the full text isn't useful,\n\t\t// just return null for anonymous exports\n\t\treturn null;\n\t}\n\treturn decl.childForFieldName(\"name\")?.text ?? null;\n}\n\n// ============================================================================\n// Per-Language Node Extractors\n// ============================================================================\n\nconst TS_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_declaration\", kind: \"function\", getName: nameField },\n\t{ type: \"method_definition\", kind: \"method\", getName: nameField },\n\t{ type: \"class_declaration\", kind: \"class\", getName: nameField },\n\t{ type: \"interface_declaration\", kind: \"interface\", getName: nameField },\n\t{ type: \"type_alias_declaration\", kind: \"type_alias\", getName: nameField },\n\t{ type: \"export_statement\", kind: \"export\", getName: exportName },\n\t{ type: \"arrow_function\", kind: \"function\", getName: arrowFunctionName },\n];\n\nconst JS_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_declaration\", kind: \"function\", getName: nameField },\n\t{ type: \"method_definition\", kind: \"method\", getName: nameField },\n\t{ type: \"class_declaration\", kind: \"class\", getName: nameField },\n\t{ type: \"export_statement\", kind: \"export\", getName: exportName },\n\t{ type: \"arrow_function\", kind: \"function\", getName: arrowFunctionName },\n];\n\nconst PYTHON_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_definition\", kind: \"function\", getName: nameField },\n\t{ type: \"class_definition\", kind: \"class\", getName: nameField },\n];\n\nconst GO_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_declaration\", kind: \"function\", getName: nameField },\n\t{ type: \"method_declaration\", kind: \"method\", getName: nameField },\n\t{ type: \"type_spec\", kind: \"struct\", getName: nameField },\n];\n\nconst RUST_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_item\", kind: \"function\", getName: nameField },\n\t{ type: \"impl_item\", kind: \"impl\", getName: (n) => n.childForFieldName(\"type\")?.text ?? null },\n\t{ type: \"struct_item\", kind: \"struct\", getName: nameField },\n\t{ type: \"enum_item\", kind: \"enum\", getName: nameField },\n\t{ type: \"trait_item\", kind: \"interface\", getName: nameField },\n];\n\nconst JAVA_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"class_declaration\", kind: \"class\", getName: nameField },\n\t{ type: \"method_declaration\", kind: \"method\", getName: nameField },\n\t{ type: \"interface_declaration\", kind: \"interface\", getName: nameField },\n];\n\nconst C_EXTRACTORS: NodeExtractor[] = [\n\t{ type: \"function_definition\", kind: \"function\", getName: cFunctionName },\n\t{ type: \"struct_specifier\", kind: \"struct\", getName: nameField },\n];\n\nconst CPP_EXTRACTORS: NodeExtractor[] = [\n\t...C_EXTRACTORS,\n\t{ type: \"class_specifier\", kind: \"class\", getName: nameField },\n];\n\nconst LANGUAGE_EXTRACTORS: Record<TreeSitterLanguage, NodeExtractor[]> = {\n\ttypescript: TS_EXTRACTORS,\n\ttsx: TS_EXTRACTORS,\n\tjavascript: JS_EXTRACTORS,\n\tpython: PYTHON_EXTRACTORS,\n\tgo: GO_EXTRACTORS,\n\trust: RUST_EXTRACTORS,\n\tjava: JAVA_EXTRACTORS,\n\tc: C_EXTRACTORS,\n\tcpp: CPP_EXTRACTORS,\n};\n\n// ============================================================================\n// Initialization\n// ============================================================================\n\n/**\n * Initialize the tree-sitter WASM runtime. Must be called before parsing.\n * Safe to call multiple times — subsequent calls are no-ops.\n */\nexport async function initTreeSitter(): Promise<void> {\n\tif (initialized) return;\n\tif (initPromise) return initPromise;\n\n\tinitPromise = (async () => {\n\t\ttry {\n\t\t\tconst mod = await import(\"web-tree-sitter\");\n\t\t\tParser = mod.Parser;\n\t\t\tLanguage = mod.Language;\n\n\t\t\tconst wasmPath = require.resolve(\"web-tree-sitter/web-tree-sitter.wasm\");\n\t\t\tconst wasmBuf = readFileSync(wasmPath);\n\t\t\tawait Parser.init({ locateFile: () => wasmPath, wasmBinary: wasmBuf });\n\t\t\tinitialized = true;\n\t\t} catch (err) {\n\t\t\t// Reset so subsequent calls can retry instead of returning\n\t\t\t// the same rejected promise forever\n\t\t\tinitPromise = null;\n\t\t\tthrow err;\n\t\t}\n\t})();\n\n\treturn initPromise;\n}\n\n// ============================================================================\n// Language Loading\n// ============================================================================\n\n/** Load and cache a tree-sitter language grammar. */\nasync function loadLanguage(lang: TreeSitterLanguage): Promise<import(\"web-tree-sitter\").Language> {\n\tconst cached = languageCache.get(lang);\n\tif (cached) return cached;\n\n\tif (!Language) {\n\t\tthrow new Error(\"tree-sitter not initialized — call initTreeSitter() first\");\n\t}\n\n\tconst grammarPath = require.resolve(GRAMMAR_PATHS[lang]);\n\tconst loaded = await Language.load(grammarPath);\n\tlanguageCache.set(lang, loaded);\n\treturn loaded;\n}\n\n// ============================================================================\n// AST Extraction\n// ============================================================================\n\n/**\n * Walk the tree and collect nodes matching the target types.\n * Returns regions sorted by start position, with nested nodes skipped\n * (only outermost matches are kept).\n */\nfunction extractRegions(rootNode: TSNode, extractors: NodeExtractor[], _sourceLines: string[]): ExtractedRegion[] {\n\t// Gather all target node types\n\tconst typeToExtractors = new Map<string, NodeExtractor>();\n\tfor (const ext of extractors) {\n\t\ttypeToExtractors.set(ext.type, ext);\n\t}\n\n\tconst targetTypes = extractors.map((e) => e.type);\n\tconst candidates = rootNode.descendantsOfType(targetTypes);\n\n\t// Convert to regions\n\tconst raw: ExtractedRegion[] = [];\n\tfor (const node of candidates) {\n\t\tconst ext = typeToExtractors.get(node.type);\n\t\tif (!ext) continue;\n\n\t\t// For struct_specifier in C, only extract if it has a body (field_declaration_list)\n\t\tif (node.type === \"struct_specifier\") {\n\t\t\tconst hasBody = node.children.some((c) => c.type === \"field_declaration_list\");\n\t\t\tif (!hasBody) continue;\n\t\t}\n\n\t\t// For arrow_function, only extract if parent is variable_declarator (named assignment)\n\t\tif (node.type === \"arrow_function\") {\n\t\t\tif (node.parent?.type !== \"variable_declarator\") continue;\n\t\t}\n\n\t\tconst startLine = node.startPosition.row + 1; // 0→1 indexed\n\t\tconst endLine = node.endPosition.row + 1;\n\n\t\traw.push({\n\t\t\tname: ext.getName(node),\n\t\t\tkind: ext.kind,\n\t\t\tstartLine,\n\t\t\tendLine,\n\t\t\tcontent: node.text,\n\t\t});\n\t}\n\n\t// Sort by start line, then by end line descending (larger ranges first)\n\traw.sort((a, b) => a.startLine - b.startLine || b.endLine - a.endLine);\n\n\t// Remove nested regions — keep only outermost\n\tconst regions: ExtractedRegion[] = [];\n\tlet lastEndLine = -1;\n\n\tfor (const region of raw) {\n\t\tif (region.startLine > lastEndLine) {\n\t\t\tregions.push(region);\n\t\t\tlastEndLine = region.endLine;\n\t\t}\n\t\t// else: this region is nested inside the previous one — skip\n\t}\n\n\treturn regions;\n}\n\n// ============================================================================\n// Gap Collection\n// ============================================================================\n\n/** Minimum number of non-blank lines for a gap to become its own chunk. */\nconst MIN_GAP_LINES = 3;\n\n/**\n * Create file-level chunks for substantial code between extracted regions.\n */\nfunction collectGaps(\n\tregions: ExtractedRegion[],\n\tsourceLines: string[],\n\tfilePath: string,\n\tfileType: TreeSitterLanguage,\n): Chunk[] {\n\tconst gaps: Chunk[] = [];\n\tlet cursor = 1; // 1-indexed current line\n\n\tfor (const region of regions) {\n\t\tif (region.startLine > cursor) {\n\t\t\tconst gapLines = sourceLines.slice(cursor - 1, region.startLine - 1);\n\t\t\tconst nonBlank = gapLines.filter((l) => l.trim().length > 0).length;\n\t\t\tif (nonBlank > MIN_GAP_LINES) {\n\t\t\t\tgaps.push({\n\t\t\t\t\tfilePath,\n\t\t\t\t\tstartLine: cursor,\n\t\t\t\t\tendLine: region.startLine - 1,\n\t\t\t\t\tkind: \"file\",\n\t\t\t\t\tname: null,\n\t\t\t\t\tcontent: gapLines.join(\"\\n\"),\n\t\t\t\t\tfileType,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\tcursor = region.endLine + 1;\n\t}\n\n\t// Trailing gap after last region\n\tif (cursor <= sourceLines.length) {\n\t\tconst gapLines = sourceLines.slice(cursor - 1);\n\t\tconst nonBlank = gapLines.filter((l) => l.trim().length > 0).length;\n\t\tif (nonBlank > MIN_GAP_LINES) {\n\t\t\tgaps.push({\n\t\t\t\tfilePath,\n\t\t\t\tstartLine: cursor,\n\t\t\t\tendLine: sourceLines.length,\n\t\t\t\tkind: \"file\",\n\t\t\t\tname: null,\n\t\t\t\tcontent: gapLines.join(\"\\n\"),\n\t\t\t\tfileType,\n\t\t\t});\n\t\t}\n\t}\n\n\treturn gaps;\n}\n\n// ============================================================================\n// Public API\n// ============================================================================\n\n/**\n * Parse a source file with tree-sitter and extract AST-aware chunks.\n *\n * Returns chunks for functions, classes, methods, and other language-specific\n * constructs, plus file-level chunks for substantial gaps between them.\n *\n * @param content - Raw source code text\n * @param filePath - Relative file path (stored in chunk metadata)\n * @param language - Tree-sitter language identifier\n */\nexport async function chunkWithTreeSitter(\n\tcontent: string,\n\tfilePath: string,\n\tlanguage: TreeSitterLanguage,\n): Promise<Chunk[]> {\n\tif (!initialized || !Parser) {\n\t\tawait initTreeSitter();\n\t}\n\n\t// After init, Parser is guaranteed to be set\n\tconst ParserCtor = Parser!;\n\tconst lang = await loadLanguage(language);\n\tconst parser = new ParserCtor();\n\tparser.setLanguage(lang);\n\n\tconst tree = parser.parse(content);\n\tif (!tree) {\n\t\t// Parse failed — free the parser WASM memory before returning\n\t\tparser.delete();\n\t\tconst lines = content.split(\"\\n\");\n\t\treturn [\n\t\t\t{\n\t\t\t\tfilePath,\n\t\t\t\tstartLine: 1,\n\t\t\t\tendLine: lines.length,\n\t\t\t\tkind: \"file\",\n\t\t\t\tname: null,\n\t\t\t\tcontent,\n\t\t\t\tfileType: language,\n\t\t\t},\n\t\t];\n\t}\n\n\ttry {\n\t\tconst sourceLines = content.split(\"\\n\");\n\t\tconst extractors = LANGUAGE_EXTRACTORS[language];\n\t\tconst regions = extractRegions(tree.rootNode, extractors, sourceLines);\n\n\t\t// Convert regions to Chunk objects\n\t\tconst chunks: Chunk[] = regions.map((r) => ({\n\t\t\tfilePath,\n\t\t\tstartLine: r.startLine,\n\t\t\tendLine: r.endLine,\n\t\t\tkind: r.kind,\n\t\t\tname: r.name,\n\t\t\tcontent: r.content,\n\t\t\tfileType: language,\n\t\t}));\n\n\t\t// Add gap chunks\n\t\tconst gaps = collectGaps(regions, sourceLines, filePath, language);\n\n\t\t// Merge and sort by start line\n\t\tconst all = [...chunks, ...gaps];\n\t\tall.sort((a, b) => a.startLine - b.startLine);\n\n\t\t// If no regions were extracted, return the whole file as one chunk\n\t\tif (chunks.length === 0) {\n\t\t\treturn [\n\t\t\t\t{\n\t\t\t\t\tfilePath,\n\t\t\t\t\tstartLine: 1,\n\t\t\t\t\tendLine: sourceLines.length,\n\t\t\t\t\tkind: \"file\",\n\t\t\t\t\tname: null,\n\t\t\t\t\tcontent,\n\t\t\t\t\tfileType: language,\n\t\t\t\t},\n\t\t\t];\n\t\t}\n\n\t\treturn all;\n\t} finally {\n\t\ttree.delete();\n\t\tparser.delete();\n\t}\n}\n"]}

package/dist/types.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Shared types for the semantic codebase search subsystem.
+ */
+/** Languages supported by tree-sitter AST chunking. */
+export type TreeSitterLanguage = "typescript" | "tsx" | "javascript" | "python" | "go" | "rust" | "java" | "c" | "cpp";
+/** Non-code file types chunked by format-specific rules. */
+export type TextFileType = "markdown" | "yaml" | "json" | "toml" | "plaintext";
+/** Union of all recognized file types. */
+export type FileType = TreeSitterLanguage | TextFileType;
+/** The kind of code construct a chunk represents. */
+export type ChunkKind = "function" | "method" | "class" | "interface" | "struct" | "enum" | "impl" | "export" | "type_alias" | "module" | "heading_section" | "top_level_key" | "paragraph" | "file";
+/** A chunk of code or text extracted from a file. */
+export interface Chunk {
+    /** Relative file path from project root. */
+    filePath: string;
+    /** 1-indexed start line in the source file. */
+    startLine: number;
+    /** 1-indexed end line (inclusive) in the source file. */
+    endLine: number;
+    /** The kind of construct this chunk represents. */
+    kind: ChunkKind;
+    /** Symbol name (function name, class name, heading text, etc.). Null for anonymous chunks. */
+    name: string | null;
+    /** The raw source text of the chunk. */
+    content: string;
+    /** Detected file type. */
+    fileType: FileType;
+}
+/** Configuration for the search index. */
+export interface IndexConfig {
+    /** Absolute path to the project root. */
+    projectRoot: string;
+    /** Absolute path to the index database directory. */
+    indexDir: string;
+    /** Absolute path to the global memory directory (e.g. ~/.dreb/memory/). */
+    globalMemoryDir?: string;
+    /** Additional directories to include in scans (bypasses gitignore). */
+    visibleDirs?: string[];
+    /** Embedding model name (used to key the embeddings table). */
+    modelName: string;
+}
+/** Stored metadata for a file in the index. */
+export interface IndexedFile {
+    id: number;
+    filePath: string;
+    mtime: number;
+    fileType: FileType;
+}
+/** A stored chunk row from the database. */
+export interface StoredChunk {
+    id: number;
+    fileId: number;
+    filePath: string;
+    startLine: number;
+    endLine: number;
+    kind: ChunkKind;
+    name: string | null;
+    content: string;
+    fileType: FileType;
+}
+/** A stored embedding row. */
+export interface StoredEmbedding {
+    chunkId: number;
+    modelName: string;
+    vector: Float32Array;
+}
+/** A single search result with scores and metadata. */
+export interface SearchResult {
+    /** The chunk this result refers to. */
+    chunk: StoredChunk;
+    /** Individual metric scores (0–1, higher is better). */
+    scores: MetricScores;
+    /** Combined rank from POEM (lower is better, 0 = top of Pareto front). */
+    rank: number;
+}
+/** Scores from each ranking metric. */
+export interface MetricScores {
+    bm25: number;
+    cosine: number;
+    pathMatch: number;
+    symbolMatch: number;
+    importGraph: number;
+    gitRecency: number;
+}
+/** Names of the 6 ranking metrics. */
+export type MetricName = keyof MetricScores;
+/** All metric names as an array for iteration. */
+export declare const METRIC_NAMES: MetricName[];
+/** Progress reporting callback for index operations. */
+export type IndexProgressCallback = (phase: string, current: number, total: number) => void;
+/** A resolved import edge: source file imports target file. */
+export interface ImportEdge {
+    /** Relative path of the importing file. */
+    source: string;
+    /** Relative path of the imported file. */
+    target: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,uDAAuD;AACvD,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG,KAAK,GAAG,YAAY,GAAG,QAAQ,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC;AAEvH,4DAA4D;AAC5D,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,WAAW,CAAC;AAE/E,0CAA0C;AAC1C,MAAM,MAAM,QAAQ,GAAG,kBAAkB,GAAG,YAAY,CAAC;AAMzD,qDAAqD;AACrD,MAAM,MAAM,SAAS,GAClB,UAAU,GACV,QAAQ,GACR,OAAO,GACP,WAAW,GACX,QAAQ,GACR,MAAM,GACN,MAAM,GACN,QAAQ,GACR,YAAY,GACZ,QAAQ,GACR,iBAAiB,GACjB,eAAe,GACf,WAAW,GACX,MAAM,CAAC;AAEV,qDAAqD;AACrD,MAAM,WAAW,KAAK;IACrB,4CAA4C;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,IAAI,EAAE,SAAS,CAAC;IAChB,8FAA8F;IAC9F,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,wCAAwC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;CACnB;AAMD,0CAA0C;AAC1C,MAAM,WAAW,WAAW;IAC3B,yCAAyC;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,QAAQ,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,WAAW,WAAW;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;CACnB;AAED,8BAA8B;AAC9B,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,CAAC;CACrB;AAMD,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC5B,uCAAuC;IACvC,KAAK,EAAE,WAAW,CAAC;IACnB,0DAAwD;IACxD,MAAM,EAAE,YAAY,CAAC;IACrB,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC;CACb;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,sCAAsC;AACtC,MAAM,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC;AAE5C,kDAAkD;AAClD,eAAO,MAAM,YAAY,EAAE,UAAU,EAAgF,CAAC;AAMtH,wDAAwD;AACxD,MAAM,MAAM,qBAAqB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;AAM5F,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IAC1B,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,MAAM,EAAE,MAAM,CAAC;CACf","sourcesContent":["/**\n * Shared types for the semantic codebase search subsystem.\n */\n\n// ============================================================================\n// Languages\n// ============================================================================\n\n/** Languages supported by tree-sitter AST chunking. */\nexport type TreeSitterLanguage = \"typescript\" | \"tsx\" | \"javascript\" | \"python\" | \"go\" | \"rust\" | \"java\" | \"c\" | \"cpp\";\n\n/** Non-code file types chunked by format-specific rules. */\nexport type TextFileType = \"markdown\" | \"yaml\" | \"json\" | \"toml\" | \"plaintext\";\n\n/** Union of all recognized file types. */\nexport type FileType = TreeSitterLanguage | TextFileType;\n\n// ============================================================================\n// Chunks\n// ============================================================================\n\n/** The kind of code construct a chunk represents. */\nexport type ChunkKind =\n\t| \"function\"\n\t| \"method\"\n\t| \"class\"\n\t| \"interface\"\n\t| \"struct\"\n\t| \"enum\"\n\t| \"impl\"\n\t| \"export\"\n\t| \"type_alias\"\n\t| \"module\"\n\t| \"heading_section\"\n\t| \"top_level_key\"\n\t| \"paragraph\"\n\t| \"file\";\n\n/** A chunk of code or text extracted from a file. */\nexport interface Chunk {\n\t/** Relative file path from project root. */\n\tfilePath: string;\n\t/** 1-indexed start line in the source file. */\n\tstartLine: number;\n\t/** 1-indexed end line (inclusive) in the source file. */\n\tendLine: number;\n\t/** The kind of construct this chunk represents. */\n\tkind: ChunkKind;\n\t/** Symbol name (function name, class name, heading text, etc.). Null for anonymous chunks. */\n\tname: string | null;\n\t/** The raw source text of the chunk. */\n\tcontent: string;\n\t/** Detected file type. */\n\tfileType: FileType;\n}\n\n// ============================================================================\n// Index\n// ============================================================================\n\n/** Configuration for the search index. */\nexport interface IndexConfig {\n\t/** Absolute path to the project root. */\n\tprojectRoot: string;\n\t/** Absolute path to the index database directory. */\n\tindexDir: string;\n\t/** Absolute path to the global memory directory (e.g. ~/.dreb/memory/). */\n\tglobalMemoryDir?: string;\n\t/** Additional directories to include in scans (bypasses gitignore). */\n\tvisibleDirs?: string[];\n\t/** Embedding model name (used to key the embeddings table). */\n\tmodelName: string;\n}\n\n/** Stored metadata for a file in the index. */\nexport interface IndexedFile {\n\tid: number;\n\tfilePath: string;\n\tmtime: number;\n\tfileType: FileType;\n}\n\n/** A stored chunk row from the database. */\nexport interface StoredChunk {\n\tid: number;\n\tfileId: number;\n\tfilePath: string;\n\tstartLine: number;\n\tendLine: number;\n\tkind: ChunkKind;\n\tname: string | null;\n\tcontent: string;\n\tfileType: FileType;\n}\n\n/** A stored embedding row. */\nexport interface StoredEmbedding {\n\tchunkId: number;\n\tmodelName: string;\n\tvector: Float32Array;\n}\n\n// ============================================================================\n// Search Results\n// ============================================================================\n\n/** A single search result with scores and metadata. */\nexport interface SearchResult {\n\t/** The chunk this result refers to. */\n\tchunk: StoredChunk;\n\t/** Individual metric scores (0–1, higher is better). */\n\tscores: MetricScores;\n\t/** Combined rank from POEM (lower is better, 0 = top of Pareto front). */\n\trank: number;\n}\n\n/** Scores from each ranking metric. */\nexport interface MetricScores {\n\tbm25: number;\n\tcosine: number;\n\tpathMatch: number;\n\tsymbolMatch: number;\n\timportGraph: number;\n\tgitRecency: number;\n}\n\n/** Names of the 6 ranking metrics. */\nexport type MetricName = keyof MetricScores;\n\n/** All metric names as an array for iteration. */\nexport const METRIC_NAMES: MetricName[] = [\"bm25\", \"cosine\", \"pathMatch\", \"symbolMatch\", \"importGraph\", \"gitRecency\"];\n\n// ============================================================================\n// Callbacks\n// ============================================================================\n\n/** Progress reporting callback for index operations. */\nexport type IndexProgressCallback = (phase: string, current: number, total: number) => void;\n\n// ============================================================================\n// Import Graph\n// ============================================================================\n\n/** A resolved import edge: source file imports target file. */\nexport interface ImportEdge {\n\t/** Relative path of the importing file. */\n\tsource: string;\n\t/** Relative path of the imported file. */\n\ttarget: string;\n}\n"]}

package/dist/types.js

@@ -0,0 +1,6 @@
+/**
+ * Shared types for the semantic codebase search subsystem.
+ */
+/** All metric names as an array for iteration. */
+export const METRIC_NAMES = ["bm25", "cosine", "pathMatch", "symbolMatch", "importGraph", "gitRecency"];
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA+HH,kDAAkD;AAClD,MAAM,CAAC,MAAM,YAAY,GAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC","sourcesContent":["/**\n * Shared types for the semantic codebase search subsystem.\n */\n\n// ============================================================================\n// Languages\n// ============================================================================\n\n/** Languages supported by tree-sitter AST chunking. */\nexport type TreeSitterLanguage = \"typescript\" | \"tsx\" | \"javascript\" | \"python\" | \"go\" | \"rust\" | \"java\" | \"c\" | \"cpp\";\n\n/** Non-code file types chunked by format-specific rules. */\nexport type TextFileType = \"markdown\" | \"yaml\" | \"json\" | \"toml\" | \"plaintext\";\n\n/** Union of all recognized file types. */\nexport type FileType = TreeSitterLanguage | TextFileType;\n\n// ============================================================================\n// Chunks\n// ============================================================================\n\n/** The kind of code construct a chunk represents. */\nexport type ChunkKind =\n\t| \"function\"\n\t| \"method\"\n\t| \"class\"\n\t| \"interface\"\n\t| \"struct\"\n\t| \"enum\"\n\t| \"impl\"\n\t| \"export\"\n\t| \"type_alias\"\n\t| \"module\"\n\t| \"heading_section\"\n\t| \"top_level_key\"\n\t| \"paragraph\"\n\t| \"file\";\n\n/** A chunk of code or text extracted from a file. */\nexport interface Chunk {\n\t/** Relative file path from project root. */\n\tfilePath: string;\n\t/** 1-indexed start line in the source file. */\n\tstartLine: number;\n\t/** 1-indexed end line (inclusive) in the source file. */\n\tendLine: number;\n\t/** The kind of construct this chunk represents. */\n\tkind: ChunkKind;\n\t/** Symbol name (function name, class name, heading text, etc.). Null for anonymous chunks. */\n\tname: string | null;\n\t/** The raw source text of the chunk. */\n\tcontent: string;\n\t/** Detected file type. */\n\tfileType: FileType;\n}\n\n// ============================================================================\n// Index\n// ============================================================================\n\n/** Configuration for the search index. */\nexport interface IndexConfig {\n\t/** Absolute path to the project root. */\n\tprojectRoot: string;\n\t/** Absolute path to the index database directory. */\n\tindexDir: string;\n\t/** Absolute path to the global memory directory (e.g. ~/.dreb/memory/). */\n\tglobalMemoryDir?: string;\n\t/** Additional directories to include in scans (bypasses gitignore). */\n\tvisibleDirs?: string[];\n\t/** Embedding model name (used to key the embeddings table). */\n\tmodelName: string;\n}\n\n/** Stored metadata for a file in the index. */\nexport interface IndexedFile {\n\tid: number;\n\tfilePath: string;\n\tmtime: number;\n\tfileType: FileType;\n}\n\n/** A stored chunk row from the database. */\nexport interface StoredChunk {\n\tid: number;\n\tfileId: number;\n\tfilePath: string;\n\tstartLine: number;\n\tendLine: number;\n\tkind: ChunkKind;\n\tname: string | null;\n\tcontent: string;\n\tfileType: FileType;\n}\n\n/** A stored embedding row. */\nexport interface StoredEmbedding {\n\tchunkId: number;\n\tmodelName: string;\n\tvector: Float32Array;\n}\n\n// ============================================================================\n// Search Results\n// ============================================================================\n\n/** A single search result with scores and metadata. */\nexport interface SearchResult {\n\t/** The chunk this result refers to. */\n\tchunk: StoredChunk;\n\t/** Individual metric scores (0–1, higher is better). */\n\tscores: MetricScores;\n\t/** Combined rank from POEM (lower is better, 0 = top of Pareto front). */\n\trank: number;\n}\n\n/** Scores from each ranking metric. */\nexport interface MetricScores {\n\tbm25: number;\n\tcosine: number;\n\tpathMatch: number;\n\tsymbolMatch: number;\n\timportGraph: number;\n\tgitRecency: number;\n}\n\n/** Names of the 6 ranking metrics. */\nexport type MetricName = keyof MetricScores;\n\n/** All metric names as an array for iteration. */\nexport const METRIC_NAMES: MetricName[] = [\"bm25\", \"cosine\", \"pathMatch\", \"symbolMatch\", \"importGraph\", \"gitRecency\"];\n\n// ============================================================================\n// Callbacks\n// ============================================================================\n\n/** Progress reporting callback for index operations. */\nexport type IndexProgressCallback = (phase: string, current: number, total: number) => void;\n\n// ============================================================================\n// Import Graph\n// ============================================================================\n\n/** A resolved import edge: source file imports target file. */\nexport interface ImportEdge {\n\t/** Relative path of the importing file. */\n\tsource: string;\n\t/** Relative path of the imported file. */\n\ttarget: string;\n}\n"]}