lmgrep 0.1.18 → 0.2.1

package/dist/infrastructure/treesitter/ChunkContextBuilder.js

@@ -0,0 +1,159 @@
+/**
+ * Builds the context header prepended to a chunk before embedding.
+ *
+ * The header is what makes an isolated function retrievable: on its own,
+ * `async function handle(req)` shares little vocabulary with "how are webhooks
+ * authenticated", but stamped with its file, enclosing class and doc comment
+ * it does. Retrieval quality depends on this at least as much as on the text.
+ */
+export class ChunkContextBuilder {
+    /** How far above a node to look for its doc comment. */
+    static MAX_COMMENT_LOOKBACK = 10;
+    /** AST node type -> the word used for it in the header. */
+    static SCOPE_KINDS = {
+        // TypeScript / JavaScript
+        class_declaration: "class",
+        class_body: "class",
+        interface_declaration: "interface",
+        module: "module",
+        namespace_declaration: "namespace",
+        // Python
+        class_definition: "class",
+        // Rust
+        impl_item: "impl",
+        trait_item: "trait",
+        mod_item: "mod",
+        // Go
+        type_declaration: "type",
+        // Ruby
+        class: "class",
+        // C / C++
+        struct_specifier: "struct",
+        class_specifier: "class",
+        namespace_definition: "namespace",
+        // Swift
+        struct_declaration: "struct",
+        extension_declaration: "extension",
+        // Scala
+        object_definition: "object",
+        trait_definition: "trait",
+        // Generic
+        ContainerDecl: "container",
+    };
+    /** Node types that declare a shape rather than behaviour. */
+    static DEFINITION_TYPES = new Set([
+        "class_declaration",
+        "class_definition",
+        "interface_declaration",
+        "type_alias_declaration",
+        "enum_declaration",
+        "struct_item",
+        "enum_item",
+        "trait_item",
+        "type_item",
+        "type_declaration",
+        "struct_specifier",
+        "class_specifier",
+        "enum_specifier",
+        "type_definition",
+        "struct_declaration",
+        "protocol_declaration",
+        "trait_definition",
+        "ContainerDecl",
+    ]);
+    /** Node types that wire things together rather than implement them. */
+    static ORCHESTRATION_TYPES = new Set([
+        "export_statement",
+        "decorated_definition",
+    ]);
+    /** The full header for one chunk. */
+    build(node, filePath, source, language) {
+        const lines = [
+            `[file: ${filePath}]`,
+            `[role: ${this.classifyRole(node)}]`,
+        ];
+        const scope = this.extractScope(node, language);
+        if (scope.length > 0) {
+            lines.push(`[scope: ${scope.map((s) => `${s.kind} ${s.name}`).join(" > ")}]`);
+        }
+        const comment = this.extractLeadingComment(node, source);
+        if (comment)
+            lines.push(`[doc: ${comment}]`);
+        return lines.join("\n");
+    }
+    /** The minimal header for a file with no parsed structure. */
+    buildFileOnly(filePath) {
+        return `[file: ${filePath}]`;
+    }
+    /** Walk up from a node collecting the named scopes that enclose it. */
+    extractScope(node, language) {
+        const scopes = [];
+        let current = node.parent;
+        while (current) {
+            if (language.scopeTypes.includes(current.type)) {
+                const name = this.nodeName(current);
+                if (name) {
+                    scopes.unshift({
+                        kind: ChunkContextBuilder.SCOPE_KINDS[current.type] ?? current.type,
+                        name,
+                    });
+                }
+            }
+            current = current.parent;
+        }
+        return scopes;
+    }
+    /**
+     * Collect the comment block or decorators immediately above a node.
+     *
+     * A blank line ends the block only once something has been collected, so a
+     * comment separated from its node by whitespace is still picked up, while
+     * unrelated code above it is not.
+     */
+    extractLeadingComment(node, source) {
+        const lines = source.split("\n");
+        const start = node.startPosition.row;
+        const collected = [];
+        for (let i = start - 1; i >= 0 && i >= start - ChunkContextBuilder.MAX_COMMENT_LOOKBACK; i--) {
+            const line = lines[i].trim();
+            if (this.looksLikeCommentOrDecorator(line)) {
+                collected.unshift(lines[i]);
+            }
+            else if (line === "") {
+                if (collected.length > 0)
+                    break;
+            }
+            else {
+                break;
+            }
+        }
+        return collected.length === 0 ? null : collected.join("\n").trim();
+    }
+    classifyRole(node) {
+        if (ChunkContextBuilder.DEFINITION_TYPES.has(node.type)) {
+            return "definition";
+        }
+        if (ChunkContextBuilder.ORCHESTRATION_TYPES.has(node.type)) {
+            return "orchestration";
+        }
+        return "implementation";
+    }
+    /** Comment syntax across every supported language, plus decorators. */
+    looksLikeCommentOrDecorator(line) {
+        return (line.startsWith("//") ||
+            line.startsWith("#") ||
+            line.startsWith("*") ||
+            line.startsWith("/*") ||
+            line.startsWith("*/") ||
+            line.startsWith("///") ||
+            line.startsWith("--") ||
+            line.startsWith("@") ||
+            line.startsWith('"""') ||
+            line.startsWith("'''"));
+    }
+    nodeName(node) {
+        return (node.childForFieldName("name") ??
+            node.children.find((c) => c.type === "identifier" || c.type === "type_identifier"))?.text;
+    }
+}
+//# sourceMappingURL=ChunkContextBuilder.js.map

package/dist/infrastructure/treesitter/ChunkContextBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChunkContextBuilder.js","sourceRoot":"","sources":["../../../src/infrastructure/treesitter/ChunkContextBuilder.ts"],"names":[],"mappings":"AAYA;;;;;;;GAOG;AACH,MAAM,OAAO,mBAAmB;IAC/B,wDAAwD;IAChD,MAAM,CAAU,oBAAoB,GAAG,EAAE,CAAC;IAElD,2DAA2D;IACnD,MAAM,CAAU,WAAW,GAA2B;QAC7D,0BAA0B;QAC1B,iBAAiB,EAAE,OAAO;QAC1B,UAAU,EAAE,OAAO;QACnB,qBAAqB,EAAE,WAAW;QAClC,MAAM,EAAE,QAAQ;QAChB,qBAAqB,EAAE,WAAW;QAClC,SAAS;QACT,gBAAgB,EAAE,OAAO;QACzB,OAAO;QACP,SAAS,EAAE,MAAM;QACjB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,KAAK;QACf,KAAK;QACL,gBAAgB,EAAE,MAAM;QACxB,OAAO;QACP,KAAK,EAAE,OAAO;QACd,UAAU;QACV,gBAAgB,EAAE,QAAQ;QAC1B,eAAe,EAAE,OAAO;QACxB,oBAAoB,EAAE,WAAW;QACjC,QAAQ;QACR,kBAAkB,EAAE,QAAQ;QAC5B,qBAAqB,EAAE,WAAW;QAClC,QAAQ;QACR,iBAAiB,EAAE,QAAQ;QAC3B,gBAAgB,EAAE,OAAO;QACzB,UAAU;QACV,aAAa,EAAE,WAAW;KAC1B,CAAC;IAEF,6DAA6D;IACrD,MAAM,CAAU,gBAAgB,GAAG,IAAI,GAAG,CAAC;QAClD,mBAAmB;QACnB,kBAAkB;QAClB,uBAAuB;QACvB,wBAAwB;QACxB,kBAAkB;QAClB,aAAa;QACb,WAAW;QACX,YAAY;QACZ,WAAW;QACX,kBAAkB;QAClB,kBAAkB;QAClB,iBAAiB;QACjB,gBAAgB;QAChB,iBAAiB;QACjB,oBAAoB;QACpB,sBAAsB;QACtB,kBAAkB;QAClB,eAAe;KACf,CAAC,CAAC;IAEH,uEAAuE;IAC/D,MAAM,CAAU,mBAAmB,GAAG,IAAI,GAAG,CAAC;QACrD,kBAAkB;QAClB,sBAAsB;KACtB,CAAC,CAAC;IAEH,qCAAqC;IACrC,KAAK,CACJ,IAAU,EACV,QAAgB,EAChB,MAAc,EACd,QAAwB;QAExB,MAAM,KAAK,GAAa;YACvB,UAAU,QAAQ,GAAG;YACrB,UAAU,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG;SACpC,CAAC;QAEF,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,KAAK,CAAC,IAAI,CACT,WAAW,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CACjE,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzD,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,GAAG,CAAC,CAAC;QAE7C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,8DAA8D;IAC9D,aAAa,CAAC,QAAgB;QAC7B,OAAO,UAAU,QAAQ,GAAG,CAAC;IAC9B,CAAC;IAED,uEAAuE;IACvE,YAAY,CAAC,IAAU,EAAE,QAAwB;QAChD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO,OAAO,EAAE,CAAC;YAChB,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpC,IAAI,IAAI,EAAE,CAAC;oBACV,MAAM,CAAC,OAAO,CAAC;wBACd,IAAI,EAAE,mBAAmB,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI;wBACnE,IAAI;qBACJ,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACH,qBAAqB,CAAC,IAAU,EAAE,MAAc;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;QACrC,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KACC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EACjB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,mBAAmB,CAAC,oBAAoB,EAC/D,CAAC,EAAE,EACF,CAAC;YACF,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;iBAAM,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM;YACjC,CAAC;iBAAM,CAAC;gBACP,MAAM;YACP,CAAC;QACF,CAAC;QAED,OAAO,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACpE,CAAC;IAED,YAAY,CAAC,IAAU;QACtB,IAAI,mBAAmB,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO,YAAY,CAAC;QACrB,CAAC;QACD,IAAI,mBAAmB,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,OAAO,eAAe,CAAC;QACxB,CAAC;QACD,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAED,uEAAuE;IAC/D,2BAA2B,CAAC,IAAY;QAC/C,OAAO,CACN,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CACtB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,IAAU;QAC1B,OAAO,CACN,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CACjB,CAAC,CAAO,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,iBAAiB,CACpE,CACD,EAAE,IAAI,CAAC;IACT,CAAC"}

package/dist/infrastructure/treesitter/EmbeddedTreeSitterAssets.d.ts

@@ -0,0 +1,8 @@
+export interface EmbeddedTreeSitterAssets {
+    parser: string;
+    grammars: Readonly<Record<string, string>>;
+}
+/** Registers paths embedded by the standalone Bun entry point. */
+export declare function registerEmbeddedTreeSitterAssets(assets: EmbeddedTreeSitterAssets): void;
+export declare function embeddedParserPath(): string | undefined;
+export declare function embeddedGrammarPath(wasmFile: string): string | undefined;

package/dist/infrastructure/treesitter/EmbeddedTreeSitterAssets.js

@@ -0,0 +1,12 @@
+let embedded;
+/** Registers paths embedded by the standalone Bun entry point. */
+export function registerEmbeddedTreeSitterAssets(assets) {
+    embedded = assets;
+}
+export function embeddedParserPath() {
+    return embedded?.parser;
+}
+export function embeddedGrammarPath(wasmFile) {
+    return embedded?.grammars[wasmFile];
+}
+//# sourceMappingURL=EmbeddedTreeSitterAssets.js.map

package/dist/infrastructure/treesitter/EmbeddedTreeSitterAssets.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EmbeddedTreeSitterAssets.js","sourceRoot":"","sources":["../../../src/infrastructure/treesitter/EmbeddedTreeSitterAssets.ts"],"names":[],"mappings":"AAKA,IAAI,QAA8C,CAAC;AAEnD,kEAAkE;AAClE,MAAM,UAAU,gCAAgC,CAC/C,MAAgC;IAEhC,QAAQ,GAAG,MAAM,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,kBAAkB;IACjC,OAAO,QAAQ,EAAE,MAAM,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IACnD,OAAO,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC"}

package/dist/infrastructure/treesitter/LanguageCatalog.d.ts

@@ -0,0 +1,30 @@
+export type LanguageId = "javascript" | "typescript" | "tsx" | "python" | "rust" | "go" | "ruby" | "c" | "cpp" | "swift" | "json" | "yaml" | "toml" | "lua" | "scala" | "zig" | "bash" | "html" | "css" | "java" | "kotlin" | "php" | "c_sharp" | "vue" | "markdown" | "nix";
+export interface LanguageConfig {
+    id: LanguageId;
+    extensions: string[];
+    wasmFile: string;
+    /** AST node types that represent top-level or nestable definitions */
+    chunkTypes: string[];
+    /** AST node types that represent import/require statements */
+    importTypes: string[];
+    /** AST node types that represent class/struct/module wrappers */
+    scopeTypes: string[];
+}
+export declare const LANGUAGES: LanguageConfig[];
+/**
+ * The grammar table and the lookups over it.
+ *
+ * Grammars ship as separate `@lumis-sh/wasm-<slug>` packages built for the
+ * current web-tree-sitter ABI, so resolution is a runtime `require.resolve`
+ * that may legitimately fail — a missing grammar means "chunk this file by
+ * sliding window", not an error.
+ */
+export declare class LanguageCatalog {
+    private readonly languages;
+    private readonly byExtension;
+    constructor(languages?: readonly LanguageConfig[]);
+    /** Grammar for a path's extension, or undefined when none is known. */
+    forFile(filePath: string): LanguageConfig | undefined;
+    /** Absolute path to a grammar's wasm, or undefined when not installed. */
+    wasmPathFor(lang: LanguageConfig): string | undefined;
+}

package/dist/{lib/chunker/languages.js → infrastructure/treesitter/LanguageCatalog.js}

@@ -1,4 +1,5 @@
 import { createRequire } from "node:module";
+import { embeddedGrammarPath } from "./EmbeddedTreeSitterAssets.js";
 const require = createRequire(import.meta.url);
 export const LANGUAGES = [
     {
@@ -141,11 +142,7 @@ export const LANGUAGES = [
             "enum_specifier",
         ],
         importTypes: ["preproc_include", "using_declaration"],
-        scopeTypes: [
-            "class_specifier",
-            "struct_specifier",
-            "namespace_definition",
-        ],
+        scopeTypes: ["class_specifier", "struct_specifier", "namespace_definition"],
     },
     {
         id: "swift",
@@ -207,11 +204,7 @@ export const LANGUAGES = [
             "trait_definition",
         ],
         importTypes: ["import_declaration"],
-        scopeTypes: [
-            "class_definition",
-            "object_definition",
-            "trait_definition",
-        ],
+        scopeTypes: ["class_definition", "object_definition", "trait_definition"],
     },
     {
         id: "zig",
@@ -346,26 +339,42 @@ export const LANGUAGES = [
         scopeTypes: ["attrset_expression", "binding_set"],
     },
 ];
-const extToLang = new Map();
-for (const lang of LANGUAGES) {
-    for (const ext of lang.extensions) {
-        extToLang.set(ext, lang);
+/**
+ * The grammar table and the lookups over it.
+ *
+ * Grammars ship as separate `@lumis-sh/wasm-<slug>` packages built for the
+ * current web-tree-sitter ABI, so resolution is a runtime `require.resolve`
+ * that may legitimately fail — a missing grammar means "chunk this file by
+ * sliding window", not an error.
+ */
+export class LanguageCatalog {
+    languages;
+    byExtension = new Map();
+    constructor(languages = LANGUAGES) {
+        this.languages = languages;
+        for (const lang of this.languages) {
+            for (const ext of lang.extensions)
+                this.byExtension.set(ext, lang);
+        }
     }
-}
-export function getLanguageForFile(filePath) {
-    const ext = filePath.slice(filePath.lastIndexOf("."));
-    return extToLang.get(ext);
-}
-export function getWasmPath(lang) {
-    // Grammars ship as per-language @lumis-sh/wasm-<slug> packages built for the
-    // current web-tree-sitter ABI. The wasm is named tree-sitter-<slug>.wasm and
-    // the slug is wasmFile minus the "tree-sitter-" prefix and ".wasm" suffix.
-    const slug = lang.wasmFile.replace(/^tree-sitter-/, "").replace(/\.wasm$/, "");
-    try {
-        return require.resolve(`@lumis-sh/wasm-${slug}/${lang.wasmFile}`);
+    /** Grammar for a path's extension, or undefined when none is known. */
+    forFile(filePath) {
+        return this.byExtension.get(filePath.slice(filePath.lastIndexOf(".")));
     }
-    catch {
-        return undefined;
+    /** Absolute path to a grammar's wasm, or undefined when not installed. */
+    wasmPathFor(lang) {
+        const bundled = embeddedGrammarPath(lang.wasmFile);
+        if (bundled)
+            return bundled;
+        const slug = lang.wasmFile
+            .replace(/^tree-sitter-/, "")
+            .replace(/\.wasm$/, "");
+        try {
+            return require.resolve(`@lumis-sh/wasm-${slug}/${lang.wasmFile}`);
+        }
+        catch {
+            return undefined;
+        }
     }
 }
-//# sourceMappingURL=languages.js.map
+//# sourceMappingURL=LanguageCatalog.js.map

package/dist/infrastructure/treesitter/LanguageCatalog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LanguageCatalog.js","sourceRoot":"","sources":["../../../src/infrastructure/treesitter/LanguageCatalog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAEpE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AA0C/C,MAAM,CAAC,MAAM,SAAS,GAAqB;IAC1C;QACC,EAAE,EAAE,YAAY;QAChB,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QAC3B,QAAQ,EAAE,6BAA6B;QACvC,UAAU,EAAE;YACX,sBAAsB;YACtB,gBAAgB;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,kBAAkB;YAClB,qBAAqB;SACrB;QACD,WAAW,EAAE,CAAC,kBAAkB,EAAE,iBAAiB,CAAC;QACpD,UAAU,EAAE,CAAC,mBAAmB,EAAE,YAAY,CAAC;KAC/C;IACD;QACC,EAAE,EAAE,YAAY;QAChB,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,6BAA6B;QACvC,UAAU,EAAE;YACX,sBAAsB;YACtB,gBAAgB;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,wBAAwB;YACxB,kBAAkB;YAClB,kBAAkB;YAClB,qBAAqB;SACrB;QACD,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,QAAQ;YACR,uBAAuB;SACvB;KACD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,sBAAsB;YACtB,gBAAgB;YAChB,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,wBAAwB;YACxB,kBAAkB;YAClB,kBAAkB;YAClB,qBAAqB;SACrB;QACD,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,QAAQ;YACR,uBAAuB;SACvB;KACD;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE;YACX,qBAAqB;YACrB,kBAAkB;YAClB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,kBAAkB,EAAE,uBAAuB,CAAC;QAC1D,UAAU,EAAE,CAAC,kBAAkB,EAAE,QAAQ,CAAC;KAC1C;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE;YACX,eAAe;YACf,WAAW;YACX,aAAa;YACb,WAAW;YACX,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,kBAAkB;SAClB;QACD,WAAW,EAAE,CAAC,iBAAiB,CAAC;QAChC,UAAU,EAAE,CAAC,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC;KACnD;IACD;QACC,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,qBAAqB;QAC/B,UAAU,EAAE;YACX,sBAAsB;YACtB,oBAAoB;YACpB,kBAAkB;YAClB,mBAAmB;YACnB,iBAAiB;SACjB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,kBAAkB,CAAC;KAChC;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,QAAQ,CAAC;QAC7D,WAAW,EAAE,CAAC,MAAM,CAAC;QACrB,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;KAC/B;IACD;QACC,EAAE,EAAE,GAAG;QACP,UAAU,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;QACxB,QAAQ,EAAE,oBAAoB;QAC9B,UAAU,EAAE;YACX,qBAAqB;YACrB,kBAAkB;YAClB,gBAAgB;YAChB,iBAAiB;YACjB,aAAa;SACb;QACD,WAAW,EAAE,CAAC,iBAAiB,CAAC;QAChC,UAAU,EAAE,CAAC,kBAAkB,CAAC;KAChC;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;QACnC,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,qBAAqB;YACrB,iBAAiB;YACjB,kBAAkB;YAClB,sBAAsB;YACtB,sBAAsB;YACtB,gBAAgB;SAChB;QACD,WAAW,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;QACrD,UAAU,EAAE,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,sBAAsB,CAAC;KAC3E;IACD;QACC,EAAE,EAAE,OAAO;QACX,UAAU,EAAE,CAAC,QAAQ,CAAC;QACtB,QAAQ,EAAE,wBAAwB;QAClC,yEAAyE;QACzE,uEAAuE;QACvE,UAAU,EAAE;YACX,sBAAsB;YACtB,mBAAmB;YACnB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,mBAAmB,CAAC;KACjC;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,QAAQ,CAAC;KACtB;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;QAC7B,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,oBAAoB,CAAC;QAClC,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,eAAe,CAAC;KAC7B;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;QAC7B,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,OAAO,CAAC;KACrB;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,2EAA2E;QAC3E,UAAU,EAAE,CAAC,sBAAsB,EAAE,qBAAqB,CAAC;QAC3D,WAAW,EAAE,CAAC,MAAM,CAAC;QACrB,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,OAAO;QACX,UAAU,EAAE,CAAC,QAAQ,CAAC;QACtB,QAAQ,EAAE,wBAAwB;QAClC,UAAU,EAAE;YACX,qBAAqB;YACrB,gBAAgB;YAChB,gBAAgB;YAChB,kBAAkB;YAClB,mBAAmB;YACnB,kBAAkB;SAClB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,kBAAkB,EAAE,mBAAmB,EAAE,kBAAkB,CAAC;KACzE;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,sBAAsB;YACtB,sBAAsB;YACtB,kBAAkB;SAClB;QACD,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC;KAC3E;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;QAC5B,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,qBAAqB,CAAC;QACnC,WAAW,EAAE,CAAC,SAAS,CAAC;QACxB,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,eAAe,CAAC;QAC1D,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,SAAS,CAAC;KACvB;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,CAAC,UAAU,EAAE,iBAAiB,EAAE,qBAAqB,CAAC;QAClE,WAAW,EAAE,CAAC,kBAAkB,CAAC;QACjC,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,MAAM;QACV,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,kBAAkB;YAClB,oBAAoB;YACpB,yBAAyB;YACzB,oBAAoB;SACpB;QACD,WAAW,EAAE,CAAC,oBAAoB,CAAC;QACnC,UAAU,EAAE,CAAC,mBAAmB,EAAE,uBAAuB,CAAC;KAC1D;IACD;QACC,EAAE,EAAE,QAAQ;QACZ,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE;YACX,mBAAmB;YACnB,oBAAoB;YACpB,sBAAsB;YACtB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,eAAe,CAAC;QAC9B,UAAU,EAAE,CAAC,mBAAmB,EAAE,oBAAoB,CAAC;KACvD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;YACnB,kBAAkB;YAClB,qBAAqB;YACrB,oBAAoB;SACpB;QACD,WAAW,EAAE,CAAC,2BAA2B,CAAC;QAC1C,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;SACnB;KACD;IACD;QACC,EAAE,EAAE,SAAS;QACb,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,QAAQ,EAAE,yBAAyB;QACnC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,kBAAkB;YAClB,oBAAoB;YACpB,oBAAoB;YACpB,yBAAyB;YACzB,sBAAsB;SACtB;QACD,WAAW,EAAE,CAAC,iBAAiB,CAAC;QAChC,UAAU,EAAE;YACX,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,uBAAuB;SACvB;KACD;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,CAAC,gBAAgB,EAAE,kBAAkB,EAAE,eAAe,CAAC;QACnE,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,EAAE;KACd;IACD;QACC,EAAE,EAAE,UAAU;QACd,UAAU,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,CAAC;QACxC,QAAQ,EAAE,2BAA2B;QACrC,wEAAwE;QACxE,yEAAyE;QACzE,wEAAwE;QACxE,UAAU,EAAE,CAAC,SAAS,CAAC;QACvB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,SAAS,CAAC;KACvB;IACD;QACC,EAAE,EAAE,KAAK;QACT,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,sBAAsB;QAChC,UAAU,EAAE,CAAC,SAAS,CAAC;QACvB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC,oBAAoB,EAAE,aAAa,CAAC;KACjD;CACD,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,OAAO,eAAe;IAIT;IAHD,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEjE,YACkB,YAAuC,SAAS;QAAhD,cAAS,GAAT,SAAS,CAAuC;QAEjE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU;gBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACpE,CAAC;IACF,CAAC;IAED,uEAAuE;IACvE,OAAO,CAAC,QAAgB;QACvB,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,0EAA0E;IAC1E,WAAW,CAAC,IAAoB;QAC/B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAE5B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ;aACxB,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;aAC5B,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC;YACJ,OAAO,OAAO,CAAC,OAAO,CAAC,kBAAkB,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnE,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}

package/dist/infrastructure/treesitter/SlidingWindowChunker.d.ts

@@ -0,0 +1,15 @@
+import { Chunk } from "../../domain/corpus/Chunk.js";
+import type { ChunkerPort } from "../../domain/ports/ChunkerPort.js";
+/**
+ * Fixed-size windows over a file, for sources with no grammar or no chunkable
+ * structure (a linear shell script, say).
+ *
+ * Windows overlap slightly so a passage straddling a boundary is still
+ * retrievable. The overlap is kept small because search deduplicates
+ * overlapping hits anyway, making a larger one pure index bloat.
+ */
+export declare class SlidingWindowChunker implements ChunkerPort {
+    private static readonly WINDOW_LINES;
+    private static readonly STRIDE_LINES;
+    chunk(filePath: string, cwd: string): Promise<Chunk[]>;
+}

package/dist/infrastructure/treesitter/SlidingWindowChunker.js

@@ -0,0 +1,42 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Chunk } from "../../domain/corpus/Chunk.js";
+import { CodeLocation } from "../../domain/corpus/CodeLocation.js";
+import { ContentHash } from "../../domain/corpus/ContentHash.js";
+/**
+ * Fixed-size windows over a file, for sources with no grammar or no chunkable
+ * structure (a linear shell script, say).
+ *
+ * Windows overlap slightly so a passage straddling a boundary is still
+ * retrievable. The overlap is kept small because search deduplicates
+ * overlapping hits anyway, making a larger one pure index bloat.
+ */
+export class SlidingWindowChunker {
+    static WINDOW_LINES = 50;
+    static STRIDE_LINES = 40;
+    async chunk(filePath, cwd) {
+        const source = readFileSync(join(cwd, filePath), "utf-8");
+        const lines = source.split("\n");
+        if (lines.length === 0)
+            return [];
+        const chunks = [];
+        for (let i = 0; i < lines.length; i += SlidingWindowChunker.STRIDE_LINES) {
+            const content = lines
+                .slice(i, i + SlidingWindowChunker.WINDOW_LINES)
+                .join("\n");
+            if (content.trim().length === 0)
+                continue;
+            const endLine = Math.min(i + SlidingWindowChunker.WINDOW_LINES, lines.length);
+            chunks.push(new Chunk({
+                location: new CodeLocation(filePath, i + 1, endLine),
+                type: "block",
+                name: `lines_${i + 1}_${endLine}`,
+                content,
+                context: `[file: ${filePath}]`,
+                hash: ContentHash.of(content),
+            }));
+        }
+        return chunks;
+    }
+}
+//# sourceMappingURL=SlidingWindowChunker.js.map

package/dist/infrastructure/treesitter/SlidingWindowChunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SlidingWindowChunker.js","sourceRoot":"","sources":["../../../src/infrastructure/treesitter/SlidingWindowChunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAGjE;;;;;;;GAOG;AACH,MAAM,OAAO,oBAAoB;IACxB,MAAM,CAAU,YAAY,GAAG,EAAE,CAAC;IAClC,MAAM,CAAU,YAAY,GAAG,EAAE,CAAC;IAE1C,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,GAAW;QACxC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,oBAAoB,CAAC,YAAY,EAAE,CAAC;YAC1E,MAAM,OAAO,GAAG,KAAK;iBACnB,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,oBAAoB,CAAC,YAAY,CAAC;iBAC/C,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAE1C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACvB,CAAC,GAAG,oBAAoB,CAAC,YAAY,EACrC,KAAK,CAAC,MAAM,CACZ,CAAC;YACF,MAAM,CAAC,IAAI,CACV,IAAI,KAAK,CAAC;gBACT,QAAQ,EAAE,IAAI,YAAY,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;gBACpD,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,EAAE;gBACjC,OAAO;gBACP,OAAO,EAAE,UAAU,QAAQ,GAAG;gBAC9B,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,OAAO,CAAC;aAC7B,CAAC,CACF,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC"}

package/dist/infrastructure/treesitter/TreeSitterChunker.d.ts

@@ -0,0 +1,42 @@
+import { Chunk } from "../../domain/corpus/Chunk.js";
+import type { ChunkerPort } from "../../domain/ports/ChunkerPort.js";
+import { ChunkContextBuilder } from "./ChunkContextBuilder.js";
+import { LanguageCatalog } from "./LanguageCatalog.js";
+import { SlidingWindowChunker } from "./SlidingWindowChunker.js";
+/**
+ * Splits source into chunks along AST boundaries.
+ *
+ * Node-bounded chunks are what make results readable: a hit is a whole
+ * function or class, not an arbitrary window through the middle of one. Files
+ * with no grammar, or no chunkable structure, fall back to a sliding window.
+ */
+export declare class TreeSitterChunker implements ChunkerPort {
+    private readonly catalog;
+    private readonly context;
+    private readonly fallback;
+    /** Chunks larger than this are split by recursing into their children. */
+    private static readonly MAX_CHUNK_TOKENS;
+    /** A one-line fragment shorter than this is noise, not a unit of code. */
+    private static readonly MIN_CHUNK_CHARS;
+    private parser;
+    private readonly loaded;
+    constructor(catalog?: LanguageCatalog, context?: ChunkContextBuilder, fallback?: SlidingWindowChunker);
+    chunk(filePath: string, cwd: string): Promise<Chunk[]>;
+    /**
+     * Markdown needs its own pass: the block grammar nests `section` nodes by
+     * heading level, so emitting them whole would collapse a document into one
+     * chunk. Splitting at every heading the grammar reports (so a `#` inside a
+     * fenced code block is not mistaken for one) gives complete,
+     * non-overlapping, heading-granular chunks instead.
+     */
+    private chunkMarkdown;
+    private collect;
+    private hasChunkableDescendants;
+    private nodeName;
+    private getParser;
+    /**
+     * Grammars are cached for the process lifetime. They are wasm modules whose
+     * linear memory only ever grows, so loading one repeatedly would leak.
+     */
+    private loadGrammar;
+}

package/dist/infrastructure/treesitter/TreeSitterChunker.js

@@ -0,0 +1,165 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Language, Parser } from "web-tree-sitter";
+import { Chunk } from "../../domain/corpus/Chunk.js";
+import { CodeLocation } from "../../domain/corpus/CodeLocation.js";
+import { ContentHash } from "../../domain/corpus/ContentHash.js";
+import { ChunkContextBuilder } from "./ChunkContextBuilder.js";
+import { embeddedParserPath } from "./EmbeddedTreeSitterAssets.js";
+import { LanguageCatalog } from "./LanguageCatalog.js";
+import { SlidingWindowChunker } from "./SlidingWindowChunker.js";
+/**
+ * Splits source into chunks along AST boundaries.
+ *
+ * Node-bounded chunks are what make results readable: a hit is a whole
+ * function or class, not an arbitrary window through the middle of one. Files
+ * with no grammar, or no chunkable structure, fall back to a sliding window.
+ */
+export class TreeSitterChunker {
+    catalog;
+    context;
+    fallback;
+    /** Chunks larger than this are split by recursing into their children. */
+    static MAX_CHUNK_TOKENS = 8192;
+    /** A one-line fragment shorter than this is noise, not a unit of code. */
+    static MIN_CHUNK_CHARS = 50;
+    parser;
+    loaded = new Map();
+    constructor(catalog = new LanguageCatalog(), context = new ChunkContextBuilder(), fallback = new SlidingWindowChunker()) {
+        this.catalog = catalog;
+        this.context = context;
+        this.fallback = fallback;
+    }
+    async chunk(filePath, cwd) {
+        const language = this.catalog.forFile(filePath);
+        if (!language)
+            return this.fallback.chunk(filePath, cwd);
+        // Parser.init() must run before any Language.load(): it bootstraps the
+        // wasm runtime that grammar loading calls into. Reversing these two
+        // lines fails with "cannot read loadWebAssemblyModule of undefined".
+        const parser = await this.getParser();
+        const grammar = await this.loadGrammar(language);
+        if (!grammar)
+            return this.fallback.chunk(filePath, cwd);
+        parser.setLanguage(grammar);
+        const source = readFileSync(join(cwd, filePath), "utf-8");
+        const tree = parser.parse(source);
+        if (!tree)
+            return this.fallback.chunk(filePath, cwd);
+        const chunks = language.id === "markdown"
+            ? this.chunkMarkdown(tree.rootNode, filePath, source)
+            : this.collect(tree.rootNode, language, filePath, source, []);
+        return chunks.length > 0 ? chunks : this.fallback.chunk(filePath, cwd);
+    }
+    /**
+     * Markdown needs its own pass: the block grammar nests `section` nodes by
+     * heading level, so emitting them whole would collapse a document into one
+     * chunk. Splitting at every heading the grammar reports (so a `#` inside a
+     * fenced code block is not mistaken for one) gives complete,
+     * non-overlapping, heading-granular chunks instead.
+     */
+    chunkMarkdown(root, filePath, source) {
+        const lines = source.split("\n");
+        const headingRows = [];
+        const visit = (node) => {
+            if (node.type === "atx_heading" || node.type === "setext_heading") {
+                headingRows.push(node.startPosition.row);
+            }
+            for (const child of node.children)
+                visit(child);
+        };
+        visit(root);
+        const starts = [...new Set([0, ...headingRows])].sort((a, b) => a - b);
+        const chunks = [];
+        for (let i = 0; i < starts.length; i++) {
+            const start = starts[i];
+            const end = i + 1 < starts.length ? starts[i + 1] : lines.length;
+            const content = lines.slice(start, end).join("\n");
+            if (content.trim().length === 0)
+                continue;
+            const heading = (lines[start] ?? "").replace(/^\s*#+\s*/, "").trim();
+            chunks.push(new Chunk({
+                location: new CodeLocation(filePath, start + 1, end),
+                type: "section",
+                name: heading.slice(0, 80) || `lines_${start + 1}_${end}`,
+                content,
+                context: this.context.buildFileOnly(filePath),
+                hash: ContentHash.of(content),
+            }));
+        }
+        return chunks;
+    }
+    collect(node, language, filePath, source, chunks) {
+        if (!language.chunkTypes.includes(node.type)) {
+            for (const child of node.children) {
+                this.collect(child, language, filePath, source, chunks);
+            }
+            return chunks;
+        }
+        const content = node.text;
+        // Too large to embed whole: descend instead, so a big class becomes its
+        // methods rather than being truncated or skipped.
+        const estimatedTokens = Math.ceil(content.length / 4);
+        if (estimatedTokens > TreeSitterChunker.MAX_CHUNK_TOKENS &&
+            this.hasChunkableDescendants(node, language)) {
+            for (const child of node.children) {
+                this.collect(child, language, filePath, source, chunks);
+            }
+            return chunks;
+        }
+        // A short one-liner carries no retrievable meaning on its own.
+        if (content.split("\n").length < 2 &&
+            content.length < TreeSitterChunker.MIN_CHUNK_CHARS) {
+            return chunks;
+        }
+        chunks.push(new Chunk({
+            location: new CodeLocation(filePath, node.startPosition.row + 1, node.endPosition.row + 1),
+            type: node.type,
+            name: this.nodeName(node) ?? `anonymous_${node.startPosition.row}`,
+            content,
+            context: this.context.build(node, filePath, source, language),
+            hash: ContentHash.of(content),
+        }));
+        return chunks;
+    }
+    hasChunkableDescendants(node, language) {
+        for (const child of node.children) {
+            if (language.chunkTypes.includes(child.type))
+                return true;
+            if (this.hasChunkableDescendants(child, language))
+                return true;
+        }
+        return false;
+    }
+    nodeName(node) {
+        return (node.childForFieldName("name") ??
+            node.children.find((c) => c.type === "identifier" ||
+                c.type === "type_identifier" ||
+                // nix binds its target via an attrpath (e.g. `outputs = ...`)
+                c.type === "attrpath"))?.text;
+    }
+    async getParser() {
+        if (!this.parser) {
+            const parserWasm = embeddedParserPath();
+            await Parser.init(parserWasm ? { locateFile: () => parserWasm } : undefined);
+            this.parser = new Parser();
+        }
+        return this.parser;
+    }
+    /**
+     * Grammars are cached for the process lifetime. They are wasm modules whose
+     * linear memory only ever grows, so loading one repeatedly would leak.
+     */
+    async loadGrammar(language) {
+        const cached = this.loaded.get(language.id);
+        if (cached)
+            return cached;
+        const wasmPath = this.catalog.wasmPathFor(language);
+        if (!wasmPath)
+            return undefined;
+        const grammar = await Language.load(wasmPath);
+        this.loaded.set(language.id, grammar);
+        return grammar;
+    }
+}
+//# sourceMappingURL=TreeSitterChunker.js.map

package/dist/infrastructure/treesitter/TreeSitterChunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TreeSitterChunker.js","sourceRoot":"","sources":["../../../src/infrastructure/treesitter/TreeSitterChunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAa,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAuB,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE;;;;;;GAMG;AACH,MAAM,OAAO,iBAAiB;IAUX;IACA;IACA;IAXlB,0EAA0E;IAClE,MAAM,CAAU,gBAAgB,GAAG,IAAI,CAAC;IAChD,0EAA0E;IAClE,MAAM,CAAU,eAAe,GAAG,EAAE,CAAC;IAErC,MAAM,CAAqB;IAClB,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEtD,YACkB,UAAU,IAAI,eAAe,EAAE,EAC/B,UAAU,IAAI,mBAAmB,EAAE,EACnC,WAAW,IAAI,oBAAoB,EAAE;QAFrC,YAAO,GAAP,OAAO,CAAwB;QAC/B,YAAO,GAAP,OAAO,CAA4B;QACnC,aAAQ,GAAR,QAAQ,CAA6B;IACpD,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,GAAW;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAEzD,uEAAuE;QACvE,oEAAoE;QACpE,qEAAqE;QACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAExD,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAE5B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAErD,MAAM,MAAM,GACX,QAAQ,CAAC,EAAE,KAAK,UAAU;YACzB,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;YACrD,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAEhE,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACxE,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,IAAU,EAAE,QAAgB,EAAE,MAAc;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEjC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,CAAC,IAAU,EAAQ,EAAE;YAClC,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;gBACnE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;gBAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,CAAC;QAEZ,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;YACjE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAE1C,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CACV,IAAI,KAAK,CAAC;gBACT,QAAQ,EAAE,IAAI,YAAY,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC;gBACpD,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,IAAI,GAAG,EAAE;gBACzD,OAAO;gBACP,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC;gBAC7C,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,OAAO,CAAC;aAC7B,CAAC,CACF,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,OAAO,CACd,IAAU,EACV,QAAwB,EACxB,QAAgB,EAChB,MAAc,EACd,MAAe;QAEf,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACzD,CAAC;YACD,OAAO,MAAM,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC;QAE1B,wEAAwE;QACxE,kDAAkD;QAClD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtD,IACC,eAAe,GAAG,iBAAiB,CAAC,gBAAgB;YACpD,IAAI,CAAC,uBAAuB,CAAC,IAAI,EAAE,QAAQ,CAAC,EAC3C,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YACzD,CAAC;YACD,OAAO,MAAM,CAAC;QACf,CAAC;QAED,+DAA+D;QAC/D,IACC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;YAC9B,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,eAAe,EACjD,CAAC;YACF,OAAO,MAAM,CAAC;QACf,CAAC;QAED,MAAM,CAAC,IAAI,CACV,IAAI,KAAK,CAAC;YACT,QAAQ,EAAE,IAAI,YAAY,CACzB,QAAQ,EACR,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,EAC1B,IAAI,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,CACxB;YACD,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,aAAa,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE;YAClE,OAAO;YACP,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;YAC7D,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,OAAO,CAAC;SAC7B,CAAC,CACF,CAAC;QACF,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,uBAAuB,CAC9B,IAAU,EACV,QAAwB;QAExB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1D,IAAI,IAAI,CAAC,uBAAuB,CAAC,KAAK,EAAE,QAAQ,CAAC;gBAAE,OAAO,IAAI,CAAC;QAChE,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IAEO,QAAQ,CAAC,IAAU;QAC1B,OAAO,CACN,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CACjB,CAAC,CAAO,EAAE,EAAE,CACX,CAAC,CAAC,IAAI,KAAK,YAAY;gBACvB,CAAC,CAAC,IAAI,KAAK,iBAAiB;gBAC5B,8DAA8D;gBAC9D,CAAC,CAAC,IAAI,KAAK,UAAU,CACtB,CACD,EAAE,IAAI,CAAC;IACT,CAAC;IAEO,KAAK,CAAC,SAAS;QACtB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;YACxC,MAAM,MAAM,CAAC,IAAI,CAChB,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,SAAS,CACzD,CAAC;YACF,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CACxB,QAAwB;QAExB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAC;QAEhC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC;IAChB,CAAC"}

package/dist/mcp.d.ts

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-import "./lib/native-tuning.js";
+import "./infrastructure/lancedb/NativeTuning.js";