@lzehrung/codegraph 2.1.0

package/dist/chunking/chunkSFC.d.ts

@@ -0,0 +1,18 @@
+import type { Chunk } from "./chunkFile.js";
+import { type SFCFramework } from "../languages/sfc.js";
+import { type LogLevel } from "../logging.js";
+export interface ChunkSFCOptions {
+    source: string;
+    filePath?: string | undefined;
+    framework: SFCFramework;
+    minTokens?: number;
+    maxTokens?: number;
+    tokenizer?: ((text: string) => number) | undefined;
+    logLevel?: LogLevel;
+}
+/**
+ * Chunks an SFC while preserving whole-file coverage: every source byte belongs
+ * to at least one semantic block chunk or bounded misc chunk.
+ */
+export declare function chunkSFCFile(opts: ChunkSFCOptions): Chunk[];
+//# sourceMappingURL=chunkSFC.d.ts.map

package/dist/chunking/chunkSFC.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSFC.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkSFC.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAK5C,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAgB,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,SAAS,EAAE,YAAY,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,eAAe,GAAG,KAAK,EAAE,CAqG3D"}

package/dist/chunking/chunkSFC.js

@@ -0,0 +1,165 @@
+import { chunkFile } from "./chunkFile.js";
+import { chunkTextFile } from "./chunkTextFile.js";
+import { withStableChunkIds } from "./chunkId.js";
+import { LANG_CONFIGS } from "../bootstrap/treeSitterLanguages.js";
+import { parseSFC, buildSvelteTemplateBlocks, scriptLanguageIdForBlock, styleLanguageKey, templateLanguageKey, } from "../languages/sfc.js";
+import { logWithLevel } from "../logging.js";
+/**
+ * Chunks an SFC while preserving whole-file coverage: every source byte belongs
+ * to at least one semantic block chunk or bounded misc chunk.
+ */
+export function chunkSFCFile(opts) {
+    const { source, filePath, framework, minTokens = 150, maxTokens = 400, tokenizer, logLevel } = opts;
+    const baseBlocks = parseSFC(source);
+    const blocks = framework === "svelte" ? [...baseBlocks, ...buildSvelteTemplateBlocks(source, baseBlocks)] : baseBlocks;
+    if (!blocks.length) {
+        return chunkTextFile({
+            source,
+            filePath,
+            languageId: framework,
+            minTokens,
+            maxTokens,
+            tokenizer,
+        });
+    }
+    const sortedBlocks = blocks.sort((a, b) => a.startOffset - b.startOffset || b.endOffset - a.endOffset);
+    const chunks = [];
+    for (const block of sortedBlocks) {
+        const blockChunks = chunkBlock({
+            block,
+            framework,
+            filePath,
+            minTokens,
+            maxTokens,
+            tokenizer,
+            ...(logLevel ? { logLevel } : {}),
+        });
+        for (const chunk of blockChunks) {
+            chunks.push({
+                ...chunk,
+                id: "",
+                startLine: chunk.startLine + block.startLine - 1,
+                endLine: chunk.endLine + block.startLine - 1,
+                type: `${block.type}:${chunk.type}`,
+                filePath: filePath ?? chunk.filePath,
+            });
+        }
+    }
+    // Semantic and text block chunkers cover every byte of non-blank block content.
+    // Keep each block's wrappers separate from inter-block text so inserting a
+    // preceding block cannot change an unchanged block's content-addressed misc IDs.
+    let coveredUntil = 0;
+    for (const block of sortedBlocks) {
+        if (block.blockStart < coveredUntil)
+            continue;
+        appendMiscChunks(chunks, source, coveredUntil, block.blockStart, framework, filePath, minTokens, maxTokens, tokenizer);
+        if (block.content.trim()) {
+            appendMiscChunks(chunks, source, block.blockStart, block.startOffset, framework, filePath, minTokens, maxTokens, tokenizer);
+            appendMiscChunks(chunks, source, block.endOffset, block.blockEnd, framework, filePath, minTokens, maxTokens, tokenizer);
+        }
+        else {
+            appendMiscChunks(chunks, source, block.blockStart, block.blockEnd, framework, filePath, minTokens, maxTokens, tokenizer);
+        }
+        coveredUntil = block.blockEnd;
+    }
+    appendMiscChunks(chunks, source, coveredUntil, source.length, framework, filePath, minTokens, maxTokens, tokenizer);
+    const sortedChunks = chunks.sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine);
+    return withStableChunkIds(sortedChunks, framework, filePath);
+}
+function appendMiscChunks(chunks, source, startOffset, endOffset, framework, filePath, minTokens, maxTokens, tokenizer) {
+    if (endOffset <= startOffset)
+        return;
+    const startLine = lineForOffset(source, startOffset);
+    const gapChunks = chunkTextFile({
+        source: source.slice(startOffset, endOffset),
+        filePath,
+        languageId: framework,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+    for (const chunk of gapChunks) {
+        chunks.push({
+            ...chunk,
+            id: "",
+            startLine: chunk.startLine + startLine - 1,
+            endLine: chunk.endLine + startLine - 1,
+            type: "misc",
+            filePath: filePath ?? chunk.filePath,
+        });
+    }
+}
+function lineForOffset(source, offset) {
+    let line = 1;
+    for (let index = 0; index < offset; index++) {
+        if (source[index] === "\n")
+            line++;
+    }
+    return line;
+}
+function chunkBlock(opts) {
+    const { block, framework, filePath, minTokens, maxTokens, tokenizer, logLevel } = opts;
+    if (!block.content.trim()) {
+        return [];
+    }
+    const languageKey = selectLanguageKey(block, framework);
+    if (languageKey) {
+        const config = LANG_CONFIGS[languageKey];
+        if (config) {
+            try {
+                return chunkFile({
+                    language: config,
+                    source: block.content,
+                    filePath,
+                    minTokens,
+                    maxTokens,
+                    tokenizer,
+                });
+            }
+            catch (error) {
+                logWithLevel(logLevel, "warn", `Warning: Semantic chunking failed for ${framework} ${block.type} block:`, error);
+            }
+        }
+    }
+    return chunkTextBlock(block, {
+        framework,
+        filePath,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+}
+function selectLanguageKey(block, framework) {
+    if (block.type === "script") {
+        const langId = scriptLanguageIdForBlock(block);
+        if (langId === "ts")
+            return "typescript";
+        if (langId === "tsx")
+            return "tsx";
+        return "javascript";
+    }
+    if (block.type === "style") {
+        const styleKey = styleLanguageKey(block);
+        if (!styleKey)
+            return null;
+        return styleKey;
+    }
+    if (block.type === "template") {
+        return templateLanguageKey(framework);
+    }
+    return null;
+}
+function chunkTextBlock(block, opts) {
+    const { framework, filePath, minTokens, maxTokens, tokenizer } = opts;
+    const languageId = `${framework}-${block.type}`;
+    const textChunks = chunkTextFile({
+        source: block.content,
+        filePath,
+        languageId,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+    return textChunks;
+}
+//# sourceMappingURL=chunkSFC.js.map

package/dist/chunking/chunkSFC.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSFC.js","sourceRoot":"","sources":["../../src/chunking/chunkSFC.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AACnE,OAAO,EACL,QAAQ,EACR,yBAAyB,EACzB,wBAAwB,EACxB,gBAAgB,EAChB,mBAAmB,GAGpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAiB,MAAM,eAAe,CAAC;AAY5D;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAqB;IAChD,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACpG,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,MAAM,GACV,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAE1G,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,aAAa,CAAC;YACnB,MAAM;YACN,QAAQ;YACR,UAAU,EAAE,SAAS;YACrB,SAAS;YACT,SAAS;YACT,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IACvG,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,UAAU,CAAC;YAC7B,KAAK;YACL,SAAS;YACT,QAAQ;YACR,SAAS;YACT,SAAS;YACT,SAAS;YACT,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClC,CAAC,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,KAAK;gBACR,EAAE,EAAE,EAAE;gBACN,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC;gBAChD,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC;gBAC5C,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE;gBACnC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ;aACrC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,2EAA2E;IAC3E,iFAAiF;IACjF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,UAAU,GAAG,YAAY;YAAE,SAAS;QAC9C,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,YAAY,EACZ,KAAK,CAAC,UAAU,EAChB,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;QACF,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACzB,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,WAAW,EACjB,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;YACF,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,QAAQ,EACd,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,QAAQ,EACd,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;QACJ,CAAC;QACD,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC;IAChC,CAAC;IACD,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEpH,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IACpH,OAAO,kBAAkB,CAAC,YAAY,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,gBAAgB,CACvB,MAAe,EACf,MAAc,EACd,WAAmB,EACnB,SAAiB,EACjB,SAAuB,EACvB,QAA4B,EAC5B,SAAiB,EACjB,SAAiB,EACjB,SAAiD;IAEjD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO;IACrC,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,aAAa,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;QAC5C,QAAQ;QACR,UAAU,EAAE,SAAS;QACrB,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC,CAAC;IACH,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,KAAK;YACR,EAAE,EAAE,EAAE;YACN,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC;YAC1C,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,SAAS,GAAG,CAAC;YACtC,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ;SACrC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,IAQnB;IACC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAEvF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACxD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;QACzC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,OAAO,SAAS,CAAC;oBACf,QAAQ,EAAE,MAAM;oBAChB,MAAM,EAAE,KAAK,CAAC,OAAO;oBACrB,QAAQ;oBACR,SAAS;oBACT,SAAS;oBACT,SAAS;iBACV,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,CACV,QAAQ,EACR,MAAM,EACN,yCAAyC,SAAS,IAAI,KAAK,CAAC,IAAI,SAAS,EACzE,KAAK,CACN,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,KAAK,EAAE;QAC3B,SAAS;QACT,QAAQ;QACR,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAe,EAAE,SAAuB;IACjE,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,YAAY,CAAC;QACzC,IAAI,MAAM,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,IAMC;IAED,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACtE,MAAM,UAAU,GAAG,GAAG,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,UAAU,GAAG,aAAa,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,OAAO;QACrB,QAAQ;QACR,UAAU;QACV,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC,CAAC;IACH,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/chunking/chunkSplit.d.ts

@@ -0,0 +1,6 @@
+import type { BlockCandidate, ChunkTokenizer, RangedChunk } from "./types.js";
+export declare function splitLargeBlockSimple(block: BlockCandidate, source: string, tokenizer: ChunkTokenizer, maxTokens: number, out: RangedChunk[], languageId: string, filePath?: string): void;
+export declare function splitLargeBlockUsingInnerBlocks(block: BlockCandidate, _innerBlocks: BlockCandidate[], source: string, tokenizer: ChunkTokenizer, maxTokens: number, out: RangedChunk[], languageId: string, _newlineOffsets: number[], filePath?: string): void;
+/** Splits an oversized string on code-point boundaries using the configured tokenizer. */
+export declare function splitTextWithinTokenBudget(text: string, tokenizer: ChunkTokenizer, maxTokens: number): string[];
+//# sourceMappingURL=chunkSplit.d.ts.map

package/dist/chunking/chunkSplit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSplit.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkSplit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9E,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,EAAE,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,IAAI,CA2DN;AAED,wBAAgB,+BAA+B,CAC7C,KAAK,EAAE,cAAc,EACrB,YAAY,EAAE,cAAc,EAAE,EAC9B,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,EAAE,EAClB,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,MAAM,EAAE,EACzB,QAAQ,CAAC,EAAE,MAAM,GAChB,IAAI,CAMN;AAED,0FAA0F;AAC1F,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C/G"}

package/dist/chunking/chunkSplit.js

@@ -0,0 +1,170 @@
+export function splitLargeBlockSimple(block, source, tokenizer, maxTokens, out, languageId, filePath) {
+    let currentStart = -1;
+    let currentEnd = -1;
+    let currentStartLine = block.startLine;
+    let currentTokens = 0;
+    const flush = () => {
+        if (currentStart === -1)
+            return;
+        emitRangeWithinBudget(block, source, currentStart, currentEnd, currentStartLine, tokenizer, maxTokens, out, languageId, filePath);
+        currentStart = -1;
+        currentEnd = -1;
+        currentTokens = 0;
+    };
+    for (const line of lineRanges(source, block.startByte, block.endByte, block.startLine)) {
+        const lineText = source.slice(line.start, line.end);
+        const lineTokens = tokenizer(lineText);
+        if (lineTokens > maxTokens) {
+            flush();
+            emitRangeWithinBudget(block, source, line.start, line.end, line.startLine, tokenizer, maxTokens, out, languageId, filePath);
+            continue;
+        }
+        if (currentStart !== -1 && currentTokens + lineTokens > maxTokens) {
+            flush();
+        }
+        if (currentStart === -1) {
+            currentStart = line.start;
+            currentStartLine = line.startLine;
+        }
+        currentEnd = line.end;
+        currentTokens += lineTokens;
+    }
+    flush();
+}
+export function splitLargeBlockUsingInnerBlocks(block, _innerBlocks, source, tokenizer, maxTokens, out, languageId, _newlineOffsets, filePath) {
+    // Semantic boundaries are useful for selecting a parent chunk, but splitting
+    // that parent at arbitrary AST offsets can make sibling line ranges overlap.
+    // Pack complete source lines here; split within a line only when that line
+    // alone exceeds the token budget.
+    splitLargeBlockSimple(block, source, tokenizer, maxTokens, out, languageId, filePath);
+}
+/** Splits an oversized string on code-point boundaries using the configured tokenizer. */
+export function splitTextWithinTokenBudget(text, tokenizer, maxTokens) {
+    if (!text.length)
+        return [];
+    if (tokenizer(text) <= maxTokens)
+        return [text];
+    const segments = [];
+    let start = 0;
+    while (start < text.length) {
+        if (tokenizer(text.slice(start)) <= maxTokens) {
+            segments.push(text.slice(start));
+            break;
+        }
+        let low = nextCodePointBoundary(text, start);
+        let high = text.length;
+        let end = start;
+        while (low <= high) {
+            const candidate = previousCodePointBoundary(text, low + ((high - low) >>> 1));
+            if (candidate <= start) {
+                low = nextCodePointBoundary(text, start);
+                continue;
+            }
+            if (tokenizer(text.slice(start, candidate)) <= maxTokens) {
+                end = candidate;
+                low = candidate + 1;
+            }
+            else {
+                high = candidate - 1;
+            }
+        }
+        if (end === start) {
+            const next = nextCodePointBoundary(text, start);
+            const singleCodePoint = text.slice(start, next);
+            if (tokenizer(singleCodePoint) > maxTokens) {
+                throw new RangeError("maxTokens is smaller than the configured tokenizer's smallest unit");
+            }
+            end = next;
+        }
+        segments.push(text.slice(start, end));
+        start = end;
+    }
+    return segments;
+}
+function emitRangeWithinBudget(block, source, start, end, startLine, tokenizer, maxTokens, out, languageId, filePath) {
+    const text = source.slice(start, end);
+    const tokenCount = tokenizer(text);
+    if (tokenCount <= maxTokens) {
+        pushRangedChunk(out, block, languageId, filePath, start, end, startLine, endLineForText(startLine, text), text, tokenCount);
+        return;
+    }
+    let segmentStart = start;
+    let segmentStartLine = startLine;
+    for (const segment of splitTextWithinTokenBudget(text, tokenizer, maxTokens)) {
+        const segmentEnd = segmentStart + segment.length;
+        pushRangedChunk(out, block, languageId, filePath, segmentStart, segmentEnd, segmentStartLine, endLineForText(segmentStartLine, segment), segment, tokenizer(segment));
+        segmentStartLine += countLineBreaks(segment);
+        segmentStart = segmentEnd;
+    }
+}
+function pushRangedChunk(out, block, languageId, filePath, sourceStart, sourceEnd, startLine, endLine, text, tokenCount) {
+    out.push({
+        id: "",
+        languageId,
+        type: block.kind,
+        startLine,
+        endLine,
+        text,
+        tokenCount,
+        sourceStart,
+        sourceEnd,
+        ...(filePath !== undefined ? { filePath } : {}),
+        ...(block.name !== undefined ? { name: block.name } : {}),
+    });
+}
+function lineRanges(source, start, end, startLine) {
+    const ranges = [];
+    let rangeStart = start;
+    let line = startLine;
+    for (let offset = start; offset < end; offset++) {
+        if (source[offset] !== "\n")
+            continue;
+        ranges.push({ start: rangeStart, end: offset + 1, startLine: line });
+        rangeStart = offset + 1;
+        line++;
+    }
+    if (rangeStart < end) {
+        ranges.push({ start: rangeStart, end, startLine: line });
+    }
+    return ranges;
+}
+function countLineBreaks(text) {
+    let count = 0;
+    for (let index = 0; index < text.length; index++) {
+        if (text[index] === "\n")
+            count++;
+    }
+    return count;
+}
+function endLineForText(startLine, text) {
+    const lineBreaks = countLineBreaks(text);
+    return startLine + lineBreaks - (text.endsWith("\n") ? 1 : 0);
+}
+function nextCodePointBoundary(text, index) {
+    if (index >= text.length)
+        return text.length;
+    const codePoint = text.codePointAt(index);
+    return index + (codePoint !== undefined && codePoint > 0xffff ? 2 : 1);
+}
+function previousCodePointBoundary(text, index) {
+    if (index <= 0 || index >= text.length || text.charCodeAt(index) < 0xdc00 || text.charCodeAt(index) > 0xdfff) {
+        return index;
+    }
+    return index - 1;
+}
+function locateLineAndColFromByte(newlineOffsets, byteOffset) {
+    let low = 0;
+    let high = newlineOffsets.length;
+    while (low < high) {
+        const mid = (low + high) >>> 1;
+        if (newlineOffsets[mid] < byteOffset)
+            low = mid + 1;
+        else
+            high = mid;
+    }
+    const line = low;
+    const prevNewline = low > 0 ? newlineOffsets[low - 1] : -1;
+    const col = byteOffset - prevNewline - 1;
+    return [line, col];
+}
+//# sourceMappingURL=chunkSplit.js.map

package/dist/chunking/chunkSplit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSplit.js","sourceRoot":"","sources":["../../src/chunking/chunkSplit.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,qBAAqB,CACnC,KAAqB,EACrB,MAAc,EACd,SAAyB,EACzB,SAAiB,EACjB,GAAkB,EAClB,UAAkB,EAClB,QAAiB;IAEjB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,gBAAgB,GAAG,KAAK,CAAC,SAAS,CAAC;IACvC,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,YAAY,KAAK,CAAC,CAAC;YAAE,OAAO;QAChC,qBAAqB,CACnB,KAAK,EACL,MAAM,EACN,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,GAAG,EACH,UAAU,EACV,QAAQ,CACT,CAAC;QACF,YAAY,GAAG,CAAC,CAAC,CAAC;QAClB,UAAU,GAAG,CAAC,CAAC,CAAC;QAChB,aAAa,GAAG,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QACpD,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;YAC3B,KAAK,EAAE,CAAC;YACR,qBAAqB,CACnB,KAAK,EACL,MAAM,EACN,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,SAAS,EACd,SAAS,EACT,SAAS,EACT,GAAG,EACH,UAAU,EACV,QAAQ,CACT,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,YAAY,KAAK,CAAC,CAAC,IAAI,aAAa,GAAG,UAAU,GAAG,SAAS,EAAE,CAAC;YAClE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;YACxB,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC;YAC1B,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,CAAC;QACD,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC;QACtB,aAAa,IAAI,UAAU,CAAC;IAC9B,CAAC;IAED,KAAK,EAAE,CAAC;AACV,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,KAAqB,EACrB,YAA8B,EAC9B,MAAc,EACd,SAAyB,EACzB,SAAiB,EACjB,GAAkB,EAClB,UAAkB,EAClB,eAAyB,EACzB,QAAiB;IAEjB,6EAA6E;IAC7E,6EAA6E;IAC7E,2EAA2E;IAC3E,kCAAkC;IAClC,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AACxF,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,0BAA0B,CAAC,IAAY,EAAE,SAAyB,EAAE,SAAiB;IACnG,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;YAC9C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,MAAM;QACR,CAAC;QAED,IAAI,GAAG,GAAG,qBAAqB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7C,IAAI,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,IAAI,GAAG,GAAG,KAAK,CAAC;QAEhB,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9E,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;gBACvB,GAAG,GAAG,qBAAqB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;gBACzD,GAAG,GAAG,SAAS,CAAC;gBAChB,GAAG,GAAG,SAAS,GAAG,CAAC,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,SAAS,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAChD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAChD,IAAI,SAAS,CAAC,eAAe,CAAC,GAAG,SAAS,EAAE,CAAC;gBAC3C,MAAM,IAAI,UAAU,CAAC,oEAAoE,CAAC,CAAC;YAC7F,CAAC;YACD,GAAG,GAAG,IAAI,CAAC;QACb,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACtC,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAqB,EACrB,MAAc,EACd,KAAa,EACb,GAAW,EACX,SAAiB,EACjB,SAAyB,EACzB,SAAiB,EACjB,GAAkB,EAClB,UAAkB,EAClB,QAA4B;IAE5B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;QAC5B,eAAe,CACb,GAAG,EACH,KAAK,EACL,UAAU,EACV,QAAQ,EACR,KAAK,EACL,GAAG,EACH,SAAS,EACT,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,EAC/B,IAAI,EACJ,UAAU,CACX,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,gBAAgB,GAAG,SAAS,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,0BAA0B,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;QAC7E,MAAM,UAAU,GAAG,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;QACjD,eAAe,CACb,GAAG,EACH,KAAK,EACL,UAAU,EACV,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC,EACzC,OAAO,EACP,SAAS,CAAC,OAAO,CAAC,CACnB,CAAC;QACF,gBAAgB,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7C,YAAY,GAAG,UAAU,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CACtB,GAAkB,EAClB,KAAqB,EACrB,UAAkB,EAClB,QAA4B,EAC5B,WAAmB,EACnB,SAAiB,EACjB,SAAiB,EACjB,OAAe,EACf,IAAY,EACZ,UAAkB;IAElB,GAAG,CAAC,IAAI,CAAC;QACP,EAAE,EAAE,EAAE;QACN,UAAU;QACV,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS;QACT,OAAO;QACP,IAAI;QACJ,UAAU;QACV,WAAW;QACX,SAAS;QACT,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,UAAU,CACjB,MAAc,EACd,KAAa,EACb,GAAW,EACX,SAAiB;IAEjB,MAAM,MAAM,GAA6D,EAAE,CAAC;IAC5E,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,IAAI,GAAG,SAAS,CAAC;IAErB,KAAK,IAAI,MAAM,GAAG,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC;QAChD,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI;YAAE,SAAS;QACtC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,EAAE,CAAC;IACT,CAAC;IAED,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,KAAK,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,IAAY;IACrD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,SAAS,GAAG,UAAU,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,KAAa;IACxD,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,KAAK,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,yBAAyB,CAAC,IAAY,EAAE,KAAa;IAC5D,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC;QAC7G,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,KAAK,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,wBAAwB,CAAC,cAAwB,EAAE,UAAkB;IAC5E,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;IACjC,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,cAAc,CAAC,GAAG,CAAE,GAAG,UAAU;YAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;;YAChD,IAAI,GAAG,GAAG,CAAC;IAClB,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC;IACjB,MAAM,WAAW,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAG,UAAU,GAAG,WAAW,GAAG,CAAC,CAAC;IACzC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AACrB,CAAC"}

package/dist/chunking/chunkTextFile.d.ts

@@ -0,0 +1,28 @@
+import type { Chunk } from "./chunkFile.js";
+import type { ChunkTokenizer } from "./types.js";
+/**
+ * Options for text file chunking (JSON, YAML, config files, etc.).
+ */
+export interface TextChunkOptions {
+    /** Text content to chunk */
+    source: string;
+    /** Optional source file path for chunk IDs */
+    filePath?: string | undefined;
+    /** Language identifier (e.g., "json", "yaml", "text") */
+    languageId?: string;
+    /** Minimum tokens per chunk (default: 150). Smaller chunks are merged. */
+    minTokens?: number;
+    /** Maximum tokens per chunk (default: 400). Larger chunks are split. */
+    maxTokens?: number;
+    /** Custom token counting function (default: whitespace-based) */
+    tokenizer?: ChunkTokenizer | undefined;
+}
+/**
+ * Splits text files into chunks respecting token budgets.
+ * Useful for JSON, YAML, configuration files, and other non-code text content.
+ *
+ * @param opts Text chunking options
+ * @returns Array of text chunks
+ */
+export declare function chunkTextFile(opts: TextChunkOptions): Chunk[];
+//# sourceMappingURL=chunkTextFile.d.ts.map

package/dist/chunking/chunkTextFile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkTextFile.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkTextFile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAI5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,SAAS,CAAC,EAAE,cAAc,GAAG,SAAS,CAAC;CACxC;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,gBAAgB,GAAG,KAAK,EAAE,CA4D7D"}

package/dist/chunking/chunkTextFile.js

@@ -0,0 +1,107 @@
+import { withStableChunkIds } from "./chunkId.js";
+import { splitTextWithinTokenBudget } from "./chunkSplit.js";
+import { countWhitespaceTokens } from "./tokenizer.js";
+/**
+ * Splits text files into chunks respecting token budgets.
+ * Useful for JSON, YAML, configuration files, and other non-code text content.
+ *
+ * @param opts Text chunking options
+ * @returns Array of text chunks
+ */
+export function chunkTextFile(opts) {
+    const { source, filePath, languageId = "text", maxTokens = 400, tokenizer = countWhitespaceTokens } = opts;
+    if (!source.length)
+        return [];
+    const chunks = [];
+    let currentStart = -1;
+    let currentEnd = -1;
+    let currentStartLine = 1;
+    let currentTokens = 0;
+    const flush = () => {
+        if (currentStart === -1)
+            return;
+        pushBoundedRange(chunks, source, currentStart, currentEnd, currentStartLine, languageId, filePath, tokenizer, maxTokens);
+        currentStart = -1;
+        currentEnd = -1;
+        currentTokens = 0;
+    };
+    for (const line of lineRanges(source)) {
+        const lineTokens = tokenizer(source.slice(line.start, line.end));
+        if (lineTokens > maxTokens) {
+            flush();
+            pushBoundedRange(chunks, source, line.start, line.end, line.startLine, languageId, filePath, tokenizer, maxTokens);
+            continue;
+        }
+        if (currentStart !== -1 && currentTokens + lineTokens > maxTokens) {
+            flush();
+        }
+        if (currentStart === -1) {
+            currentStart = line.start;
+            currentStartLine = line.startLine;
+        }
+        currentEnd = line.end;
+        currentTokens += lineTokens;
+    }
+    flush();
+    return withStableChunkIds(chunks, languageId, filePath);
+}
+function pushBoundedRange(chunks, source, start, end, startLine, languageId, filePath, tokenizer, maxTokens) {
+    const text = source.slice(start, end);
+    const tokenCount = tokenizer(text);
+    if (tokenCount <= maxTokens) {
+        chunks.push({
+            id: "",
+            languageId,
+            type: "text",
+            startLine,
+            endLine: endLineForText(startLine, text),
+            text,
+            tokenCount,
+            ...(filePath !== undefined ? { filePath } : {}),
+        });
+        return;
+    }
+    let segmentStartLine = startLine;
+    for (const segment of splitTextWithinTokenBudget(text, tokenizer, maxTokens)) {
+        chunks.push({
+            id: "",
+            languageId,
+            type: "text",
+            startLine: segmentStartLine,
+            endLine: endLineForText(segmentStartLine, segment),
+            text: segment,
+            tokenCount: tokenizer(segment),
+            ...(filePath !== undefined ? { filePath } : {}),
+        });
+        segmentStartLine += countLineBreaks(segment);
+    }
+}
+function lineRanges(source) {
+    const ranges = [];
+    let start = 0;
+    let line = 1;
+    for (let offset = 0; offset < source.length; offset++) {
+        if (source[offset] !== "\n")
+            continue;
+        ranges.push({ start, end: offset + 1, startLine: line });
+        start = offset + 1;
+        line++;
+    }
+    if (start < source.length) {
+        ranges.push({ start, end: source.length, startLine: line });
+    }
+    return ranges;
+}
+function countLineBreaks(text) {
+    let count = 0;
+    for (let index = 0; index < text.length; index++) {
+        if (text[index] === "\n")
+            count++;
+    }
+    return count;
+}
+function endLineForText(startLine, text) {
+    const lineBreaks = countLineBreaks(text);
+    return startLine + lineBreaks - (text.endsWith("\n") ? 1 : 0);
+}
+//# sourceMappingURL=chunkTextFile.js.map

package/dist/chunking/chunkTextFile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkTextFile.js","sourceRoot":"","sources":["../../src/chunking/chunkTextFile.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAqBvD;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,IAAsB;IAClD,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,GAAG,MAAM,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,GAAG,qBAAqB,EAAE,GAAG,IAAI,CAAC;IAC3G,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,YAAY,KAAK,CAAC,CAAC;YAAE,OAAO;QAChC,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,QAAQ,EACR,SAAS,EACT,SAAS,CACV,CAAC;QACF,YAAY,GAAG,CAAC,CAAC,CAAC;QAClB,UAAU,GAAG,CAAC,CAAC,CAAC;QAChB,aAAa,GAAG,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;YAC3B,KAAK,EAAE,CAAC;YACR,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,SAAS,EACd,UAAU,EACV,QAAQ,EACR,SAAS,EACT,SAAS,CACV,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,YAAY,KAAK,CAAC,CAAC,IAAI,aAAa,GAAG,UAAU,GAAG,SAAS,EAAE,CAAC;YAClE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;YACxB,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC;YAC1B,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,CAAC;QACD,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC;QACtB,aAAa,IAAI,UAAU,CAAC;IAC9B,CAAC;IAED,KAAK,EAAE,CAAC;IACR,OAAO,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,gBAAgB,CACvB,MAAe,EACf,MAAc,EACd,KAAa,EACb,GAAW,EACX,SAAiB,EACjB,UAAkB,EAClB,QAA4B,EAC5B,SAAyB,EACzB,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,EAAE;YACN,UAAU;YACV,IAAI,EAAE,MAAM;YACZ,SAAS;YACT,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC;YACxC,IAAI;YACJ,UAAU;YACV,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,IAAI,gBAAgB,GAAG,SAAS,CAAC;IACjC,KAAK,MAAM,OAAO,IAAI,0BAA0B,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,EAAE;YACN,UAAU;YACV,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,gBAAgB;YAC3B,OAAO,EAAE,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC;YAClD,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC;YAC9B,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,gBAAgB,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,MAAM,MAAM,GAA6D,EAAE,CAAC;IAC5E,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACtD,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI;YAAE,SAAS;QACtC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;QACnB,IAAI,EAAE,CAAC;IACT,CAAC;IAED,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,KAAK,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,IAAY;IACrD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,SAAS,GAAG,UAAU,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC"}

package/dist/chunking/languageConfig.d.ts

@@ -0,0 +1,35 @@
+import type { LanguageDefinition } from "../languages/types.js";
+/**
+ * Configuration for a specific language's chunking behavior.
+ * Contains Tree-sitter parser, query, and capture group definitions.
+ */
+export interface LanguageConfig {
+    /** Language identifier */
+    id: string;
+    /** Codegraph language id used by native query execution */
+    supportId: string;
+    /** Generated query text used for chunk extraction */
+    queryText: string;
+    /** Original language definition used for JS fallback */
+    definition: LanguageDefinition;
+    /** Capture group names used in queries */
+    captures: {
+        /** Capture name for symbol names (e.g., function names) */
+        name: string;
+        /** Prefix for semantic block captures */
+        blockPrefix: string;
+        /** Capture name for inner control-flow blocks */
+        innerBlock: string;
+        /** Capture names for comments and docstrings */
+        comments: string[];
+    };
+}
+/**
+ * Creates a language configuration for semantic chunking.
+ *
+ * @param def Language definition
+ * @param publicId Public language id exposed by chunking output and lookup tables
+ * @returns Language configuration object
+ */
+export declare function makeLanguageConfig(def: LanguageDefinition, publicId?: string): LanguageConfig;
+//# sourceMappingURL=languageConfig.d.ts.map

package/dist/chunking/languageConfig.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"languageConfig.d.ts","sourceRoot":"","sources":["../../src/chunking/languageConfig.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAGhE;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,UAAU,EAAE,kBAAkB,CAAC;IAC/B,0CAA0C;IAC1C,QAAQ,EAAE;QACR,2DAA2D;QAC3D,IAAI,EAAE,MAAM,CAAC;QACb,yCAAyC;QACzC,WAAW,EAAE,MAAM,CAAC;QACpB,iDAAiD;QACjD,UAAU,EAAE,MAAM,CAAC;QACnB,gDAAgD;QAChD,QAAQ,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;CACH;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,kBAAkB,EAAE,QAAQ,SAAS,GAAG,cAAc,CAe7F"}

package/dist/chunking/languageConfig.js

@@ -0,0 +1,24 @@
+import { generateChunkingQuery } from "../languages/queryGenerator.js";
+/**
+ * Creates a language configuration for semantic chunking.
+ *
+ * @param def Language definition
+ * @param publicId Public language id exposed by chunking output and lookup tables
+ * @returns Language configuration object
+ */
+export function makeLanguageConfig(def, publicId = def.id) {
+    const queryText = generateChunkingQuery(def);
+    return {
+        id: publicId,
+        supportId: def.id,
+        queryText,
+        definition: def,
+        captures: {
+            name: "chunk.name",
+            blockPrefix: "chunk.block.",
+            innerBlock: "chunk.block.inner",
+            comments: ["chunk.comment", "chunk.docstring"],
+        },
+    };
+}
+//# sourceMappingURL=languageConfig.js.map

package/dist/chunking/languageConfig.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"languageConfig.js","sourceRoot":"","sources":["../../src/chunking/languageConfig.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AA4BvE;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAuB,EAAE,QAAQ,GAAG,GAAG,CAAC,EAAE;IAC3E,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAE7C,OAAO;QACL,EAAE,EAAE,QAAQ;QACZ,SAAS,EAAE,GAAG,CAAC,EAAE;QACjB,SAAS;QACT,UAAU,EAAE,GAAG;QACf,QAAQ,EAAE;YACR,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,cAAc;YAC3B,UAAU,EAAE,mBAAmB;YAC/B,QAAQ,EAAE,CAAC,eAAe,EAAE,iBAAiB,CAAC;SAC/C;KACF,CAAC;AACJ,CAAC"}

package/dist/chunking/tokenizer.d.ts

@@ -0,0 +1,2 @@
+export declare function countWhitespaceTokens(text: string): number;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/chunking/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/chunking/tokenizer.ts"],"names":[],"mappings":"AAAA,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG1D"}

package/dist/chunking/tokenizer.js

@@ -0,0 +1,6 @@
+export function countWhitespaceTokens(text) {
+    if (!text.trim())
+        return 0;
+    return text.trim().split(/\s+/).length;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/chunking/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/chunking/tokenizer.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AACzC,CAAC"}