@lzehrung/codegraph-core 2.1.0

package/dist/chunking/chunkId.js

@@ -0,0 +1,30 @@
+import { createHash } from "node:crypto";
+/**
+ * Assigns content-addressed IDs after chunking has finalized each chunk's text.
+ * Identical chunks use a deterministic suffix only to preserve uniqueness.
+ */
+export function withStableChunkIds(chunks, languageId, filePath) {
+    const occurrences = new Map();
+    const scopedPath = filePath ?? "unknown";
+    return chunks.map((chunk) => {
+        const digest = createHash("sha256")
+            .update("codegraph-chunk-id-v1\0")
+            .update(languageId)
+            .update("\0")
+            .update(scopedPath)
+            .update("\0")
+            .update(chunk.type)
+            .update("\0")
+            .update(chunk.name ?? "")
+            .update("\0")
+            .update(chunk.text)
+            .digest("hex");
+        const occurrence = occurrences.get(digest) ?? 0;
+        occurrences.set(digest, occurrence + 1);
+        const id = occurrence === 0
+            ? `${languageId}:${scopedPath}:${digest}`
+            : `${languageId}:${scopedPath}:${digest}:${occurrence}`;
+        return { ...chunk, id };
+    });
+}
+//# sourceMappingURL=chunkId.js.map

package/dist/chunking/chunkId.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkId.js","sourceRoot":"","sources":["../../src/chunking/chunkId.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe,EAAE,UAAkB,EAAE,QAA4B;IAClG,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,UAAU,GAAG,QAAQ,IAAI,SAAS,CAAC;IAEzC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;aAChC,MAAM,CAAC,yBAAyB,CAAC;aACjC,MAAM,CAAC,UAAU,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,UAAU,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;aACxB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,KAAK,CAAC,CAAC;QACjB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAChD,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,EAAE,GACN,UAAU,KAAK,CAAC;YACd,CAAC,CAAC,GAAG,UAAU,IAAI,UAAU,IAAI,MAAM,EAAE;YACzC,CAAC,CAAC,GAAG,UAAU,IAAI,UAAU,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;QAE5D,OAAO,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/chunking/chunkMatches.d.ts

@@ -0,0 +1,4 @@
+import type { LanguageConfig } from "./languageConfig.js";
+import type { ChunkMatch } from "./types.js";
+export declare function getChunkMatches(language: LanguageConfig, source: string, _filePath?: string | undefined): ChunkMatch[];
+//# sourceMappingURL=chunkMatches.d.ts.map

package/dist/chunking/chunkMatches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkMatches.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkMatches.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAG1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,cAAc,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAC7B,UAAU,EAAE,CAYd"}

package/dist/chunking/chunkMatches.js

@@ -0,0 +1,65 @@
+import { Buffer } from "node:buffer";
+import { supportById } from "../languages.js";
+import { getNativeSingleQueryExecution } from "../native/treeSitterNative.js";
+export function getChunkMatches(language, source, _filePath) {
+    const support = supportById(language.supportId);
+    if (support) {
+        const nativeExecution = getNativeSingleQueryExecution(source, support, language.queryText);
+        if (nativeExecution.matches) {
+            // Native Tree-sitter indexes are UTF-8 bytes; JS String.slice uses UTF-16 units.
+            const toStringIndex = createUtf8ByteToStringIndex(source);
+            return nativeExecution.matches.map((match) => toChunkMatchFromNative(match, toStringIndex));
+        }
+    }
+    return [];
+}
+function toChunkMatchFromNative(match, toStringIndex) {
+    return {
+        captures: match.captures.map((capture) => ({
+            name: capture.name,
+            text: capture.text,
+            startByte: toStringIndex(capture.start.index),
+            endByte: toStringIndex(capture.end.index),
+            startLine: capture.start.row + 1,
+            endLine: capture.end.row + 1,
+            nodeType: capture.nodeType,
+        })),
+    };
+}
+function createUtf8ByteToStringIndex(source) {
+    const utf8Length = Buffer.byteLength(source, "utf8");
+    if (utf8Length === source.length) {
+        return (byteOffset) => Math.max(0, Math.min(byteOffset, source.length));
+    }
+    const byteToStringIndex = new Uint32Array(utf8Length + 1);
+    let byteOffset = 0;
+    let stringIndex = 0;
+    while (stringIndex < source.length) {
+        const codePoint = source.codePointAt(stringIndex);
+        if (codePoint === undefined)
+            break;
+        const charStringLength = codePoint > 0xffff ? 2 : 1;
+        const charByteLength = utf8ByteLengthForCodePoint(codePoint);
+        for (let offset = 1; offset < charByteLength; offset += 1) {
+            byteToStringIndex[byteOffset + offset] = stringIndex;
+        }
+        byteOffset += charByteLength;
+        stringIndex += charStringLength;
+        byteToStringIndex[byteOffset] = stringIndex;
+    }
+    byteToStringIndex[byteOffset] = source.length;
+    return (index) => {
+        const bounded = Math.max(0, Math.min(index, byteToStringIndex.length - 1));
+        return byteToStringIndex[bounded] ?? source.length;
+    };
+}
+function utf8ByteLengthForCodePoint(codePoint) {
+    if (codePoint <= 0x7f)
+        return 1;
+    if (codePoint <= 0x7ff)
+        return 2;
+    if (codePoint <= 0xffff)
+        return 3;
+    return 4;
+}
+//# sourceMappingURL=chunkMatches.js.map

package/dist/chunking/chunkMatches.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkMatches.js","sourceRoot":"","sources":["../../src/chunking/chunkMatches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,6BAA6B,EAAoB,MAAM,+BAA+B,CAAC;AAGhG,MAAM,UAAU,eAAe,CAC7B,QAAwB,EACxB,MAAc,EACd,SAA8B;IAE9B,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,eAAe,GAAG,6BAA6B,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC3F,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YAC5B,iFAAiF;YACjF,MAAM,aAAa,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;YAC1D,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAkB,EAAE,aAA6C;IAC/F,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACzC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,SAAS,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7C,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;YACzC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;YAChC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,MAAc;IACjD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,IAAI,UAAU,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACjC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAC1D,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,SAAS,KAAK,SAAS;YAAE,MAAM;QAEnC,MAAM,gBAAgB,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,0BAA0B,CAAC,SAAS,CAAC,CAAC;QAE7D,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YAC1D,iBAAiB,CAAC,UAAU,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC;QACvD,CAAC;QAED,UAAU,IAAI,cAAc,CAAC;QAC7B,WAAW,IAAI,gBAAgB,CAAC;QAChC,iBAAiB,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;IAC9C,CAAC;IAED,iBAAiB,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAE9C,OAAO,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,iBAAiB,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;IACrD,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,SAAiB;IACnD,IAAI,SAAS,IAAI,IAAI;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,SAAS,IAAI,KAAK;QAAE,OAAO,CAAC,CAAC;IACjC,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/chunking/chunkMerge.d.ts

@@ -0,0 +1,4 @@
+import type { ChunkTokenizer, RangedChunk } from "./types.js";
+export declare function mergeSmallChunks(chunks: RangedChunk[], minTokens: number, maxTokens: number, tokenizer: ChunkTokenizer): RangedChunk[];
+export declare function fillGapsWithMiscChunks(chunks: RangedChunk[], source: string, languageId: string, filePath: string | undefined, tokenizer: ChunkTokenizer, minTokens: number, maxTokens: number, newlineOffsets: number[]): RangedChunk[];
+//# sourceMappingURL=chunkMerge.d.ts.map

package/dist/chunking/chunkMerge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkMerge.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkMerge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAkB,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9E,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,WAAW,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,cAAc,GACxB,WAAW,EAAE,CAkDf;AAED,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,WAAW,EAAE,EACrB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,MAAM,EAAE,GACvB,WAAW,EAAE,CAmEf"}

package/dist/chunking/chunkMerge.js

@@ -0,0 +1,144 @@
+import { splitLargeBlockSimple } from "./chunkSplit.js";
+export function mergeSmallChunks(chunks, minTokens, maxTokens, tokenizer) {
+    if (!chunks.length)
+        return [];
+    const merged = [];
+    let index = 0;
+    while (index < chunks.length) {
+        let current = { ...chunks[index] };
+        index++;
+        while (current.tokenCount < minTokens && index < chunks.length) {
+            if (current.type === "misc" && current.tokenCount === 0)
+                break;
+            const next = chunks[index];
+            // Hierarchical chunks may overlap, but nested ranges must remain separate.
+            // Only genuinely adjacent ranges can be concatenated without duplicating
+            // source text inside the resulting chunk.
+            if (next.sourceStart < current.sourceEnd)
+                break;
+            if (next.sourceStart !== current.sourceEnd)
+                break;
+            const combinedText = `${current.text}${next.text}`;
+            const combinedTokens = tokenizer(combinedText);
+            if (combinedTokens > maxTokens)
+                break;
+            const resolvedName = current.name ?? next.name;
+            let type = current.type;
+            if (current.type === "misc" && current.tokenCount === 0) {
+                type = next.type;
+            }
+            else if (next.type === "misc" && next.tokenCount === 0) {
+                type = current.type;
+            }
+            else if (current.type !== next.type) {
+                type = `${current.type}+${next.type}`;
+            }
+            current = {
+                ...current,
+                endLine: next.endLine,
+                sourceEnd: next.sourceEnd,
+                text: combinedText,
+                tokenCount: combinedTokens,
+                type,
+                ...(resolvedName !== undefined ? { name: resolvedName } : {}),
+            };
+            index++;
+        }
+        merged.push(current);
+    }
+    return merged;
+}
+export function fillGapsWithMiscChunks(chunks, source, languageId, filePath, tokenizer, minTokens, maxTokens, newlineOffsets) {
+    if (!source.length)
+        return [];
+    const result = [];
+    let currentOffset = 0;
+    const sortedChunks = [...chunks].sort((left, right) => left.sourceStart - right.sourceStart || right.sourceEnd - left.sourceEnd);
+    for (const chunk of sortedChunks) {
+        if (chunk.sourceStart > currentOffset) {
+            const appendedToPrevious = appendTriviaGap(result, source, currentOffset, chunk.sourceStart, tokenizer, maxTokens, newlineOffsets);
+            if (!appendedToPrevious) {
+                pushMiscRange(result, source, currentOffset, chunk.sourceStart, languageId, filePath, tokenizer, maxTokens, newlineOffsets);
+            }
+        }
+        // Keep nested declarations as independent chunks. They already cover bytes
+        // inside the parent, so they must not cause another misc chunk to be emitted.
+        result.push(chunk);
+        currentOffset = Math.max(currentOffset, chunk.sourceEnd);
+    }
+    if (currentOffset < source.length) {
+        const appendedToPrevious = appendTriviaGap(result, source, currentOffset, source.length, tokenizer, maxTokens, newlineOffsets);
+        if (!appendedToPrevious) {
+            pushMiscRange(result, source, currentOffset, source.length, languageId, filePath, tokenizer, maxTokens, newlineOffsets);
+        }
+    }
+    return mergeSmallChunks(result, minTokens, maxTokens, tokenizer);
+}
+function appendTriviaGap(chunks, source, start, end, tokenizer, maxTokens, newlineOffsets) {
+    if (!chunks.length || end <= start)
+        return false;
+    const previous = chunks[chunks.length - 1];
+    if (previous.sourceEnd !== start)
+        return false;
+    const text = source.slice(start, end);
+    if (!/^[\s;]+$/u.test(text))
+        return false;
+    const combinedText = `${previous.text}${text}`;
+    const tokenCount = tokenizer(combinedText);
+    if (tokenCount > maxTokens)
+        return false;
+    const [endRow] = locateLineFromOffset(newlineOffsets, Math.max(start, end - 1));
+    chunks[chunks.length - 1] = {
+        ...previous,
+        endLine: endRow + 1,
+        sourceEnd: end,
+        text: combinedText,
+        tokenCount,
+    };
+    return true;
+}
+function pushMiscRange(out, source, start, end, languageId, filePath, tokenizer, maxTokens, newlineOffsets) {
+    if (end <= start)
+        return;
+    const [startRow] = locateLineFromOffset(newlineOffsets, start);
+    const [endRow] = locateLineFromOffset(newlineOffsets, Math.max(start, end - 1));
+    const block = {
+        kind: "misc",
+        startByte: start,
+        endByte: end,
+        startLine: startRow + 1,
+        endLine: endRow + 1,
+    };
+    const text = source.slice(start, end);
+    const tokenCount = tokenizer(text);
+    if (tokenCount <= maxTokens) {
+        out.push({
+            id: "",
+            languageId,
+            type: "misc",
+            startLine: block.startLine,
+            endLine: block.endLine,
+            text,
+            tokenCount,
+            sourceStart: start,
+            sourceEnd: end,
+            ...(filePath !== undefined ? { filePath } : {}),
+        });
+        return;
+    }
+    splitLargeBlockSimple(block, source, tokenizer, maxTokens, out, languageId, filePath);
+}
+function locateLineFromOffset(newlineOffsets, offset) {
+    let low = 0;
+    let high = newlineOffsets.length;
+    while (low < high) {
+        const mid = (low + high) >>> 1;
+        if (newlineOffsets[mid] < offset)
+            low = mid + 1;
+        else
+            high = mid;
+    }
+    const previousNewline = low > 0 ? newlineOffsets[low - 1] : -1;
+    return [low, offset - previousNewline - 1];
+}
+//# sourceMappingURL=chunkMerge.js.map

package/dist/chunking/chunkMerge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkMerge.js","sourceRoot":"","sources":["../../src/chunking/chunkMerge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAGxD,MAAM,UAAU,gBAAgB,CAC9B,MAAqB,EACrB,SAAiB,EACjB,SAAiB,EACjB,SAAyB;IAEzB,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,OAAO,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAE,EAAE,CAAC;QACpC,KAAK,EAAE,CAAC;QAER,OAAO,OAAO,CAAC,UAAU,GAAG,SAAS,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC/D,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,CAAC;gBAAE,MAAM;YAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;YAE5B,2EAA2E;YAC3E,yEAAyE;YACzE,0CAA0C;YAC1C,IAAI,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,SAAS;gBAAE,MAAM;YAChD,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,CAAC,SAAS;gBAAE,MAAM;YAClD,MAAM,YAAY,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,MAAM,cAAc,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,IAAI,cAAc,GAAG,SAAS;gBAAE,MAAM;YAEtC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;YAE/C,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YACxB,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBACxD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBACzD,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YACtB,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtC,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACxC,CAAC;YAED,OAAO,GAAG;gBACR,GAAG,OAAO;gBACV,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,cAAc;gBAC1B,IAAI;gBACJ,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9D,CAAC;YACF,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,MAAqB,EACrB,MAAc,EACd,UAAkB,EAClB,QAA4B,EAC5B,SAAyB,EACzB,SAAiB,EACjB,SAAiB,EACjB,cAAwB;IAExB,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CACnC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAC1F,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,WAAW,GAAG,aAAa,EAAE,CAAC;YACtC,MAAM,kBAAkB,GAAG,eAAe,CACxC,MAAM,EACN,MAAM,EACN,aAAa,EACb,KAAK,CAAC,WAAW,EACjB,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;YACF,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,aAAa,CACX,MAAM,EACN,MAAM,EACN,aAAa,EACb,KAAK,CAAC,WAAW,EACjB,UAAU,EACV,QAAQ,EACR,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;YACJ,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,kBAAkB,GAAG,eAAe,CACxC,MAAM,EACN,MAAM,EACN,aAAa,EACb,MAAM,CAAC,MAAM,EACb,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;QACF,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,aAAa,CACX,MAAM,EACN,MAAM,EACN,aAAa,EACb,MAAM,CAAC,MAAM,EACb,UAAU,EACV,QAAQ,EACR,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,eAAe,CACtB,MAAqB,EACrB,MAAc,EACd,KAAa,EACb,GAAW,EACX,SAAyB,EACzB,SAAiB,EACjB,cAAwB;IAExB,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAEjD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IAC5C,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1C,MAAM,YAAY,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;IAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;IAC3C,IAAI,UAAU,GAAG,SAAS;QAAE,OAAO,KAAK,CAAC;IAEzC,MAAM,CAAC,MAAM,CAAC,GAAG,oBAAoB,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;QAC1B,GAAG,QAAQ;QACX,OAAO,EAAE,MAAM,GAAG,CAAC;QACnB,SAAS,EAAE,GAAG;QACd,IAAI,EAAE,YAAY;QAClB,UAAU;KACX,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,GAAkB,EAClB,MAAc,EACd,KAAa,EACb,GAAW,EACX,UAAkB,EAClB,QAA4B,EAC5B,SAAyB,EACzB,SAAiB,EACjB,cAAwB;IAExB,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO;IAEzB,MAAM,CAAC,QAAQ,CAAC,GAAG,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAC/D,MAAM,CAAC,MAAM,CAAC,GAAG,oBAAoB,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAmB;QAC5B,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,GAAG;QACZ,SAAS,EAAE,QAAQ,GAAG,CAAC;QACvB,OAAO,EAAE,MAAM,GAAG,CAAC;KACpB,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEnC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,EAAE;YACN,UAAU;YACV,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI;YACJ,UAAU;YACV,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,GAAG;YACd,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,oBAAoB,CAAC,cAAwB,EAAE,MAAc;IACpE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;IACjC,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,cAAc,CAAC,GAAG,CAAE,GAAG,MAAM;YAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;;YAC5C,IAAI,GAAG,GAAG,CAAC;IAClB,CAAC;IACD,MAAM,eAAe,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC"}

package/dist/chunking/chunkSFC.d.ts

@@ -0,0 +1,18 @@
+import type { Chunk } from "./chunkFile.js";
+import { type SFCFramework } from "../languages/sfc.js";
+import { type LogLevel } from "../logging.js";
+export interface ChunkSFCOptions {
+    source: string;
+    filePath?: string | undefined;
+    framework: SFCFramework;
+    minTokens?: number;
+    maxTokens?: number;
+    tokenizer?: ((text: string) => number) | undefined;
+    logLevel?: LogLevel;
+}
+/**
+ * Chunks an SFC while preserving whole-file coverage: every source byte belongs
+ * to at least one semantic block chunk or bounded misc chunk.
+ */
+export declare function chunkSFCFile(opts: ChunkSFCOptions): Chunk[];
+//# sourceMappingURL=chunkSFC.d.ts.map

package/dist/chunking/chunkSFC.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSFC.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkSFC.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAK5C,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAgB,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,SAAS,EAAE,YAAY,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,eAAe,GAAG,KAAK,EAAE,CAqG3D"}

package/dist/chunking/chunkSFC.js

@@ -0,0 +1,165 @@
+import { chunkFile } from "./chunkFile.js";
+import { chunkTextFile } from "./chunkTextFile.js";
+import { withStableChunkIds } from "./chunkId.js";
+import { LANG_CONFIGS } from "../bootstrap/treeSitterLanguages.js";
+import { parseSFC, buildSvelteTemplateBlocks, scriptLanguageIdForBlock, styleLanguageKey, templateLanguageKey, } from "../languages/sfc.js";
+import { logWithLevel } from "../logging.js";
+/**
+ * Chunks an SFC while preserving whole-file coverage: every source byte belongs
+ * to at least one semantic block chunk or bounded misc chunk.
+ */
+export function chunkSFCFile(opts) {
+    const { source, filePath, framework, minTokens = 150, maxTokens = 400, tokenizer, logLevel } = opts;
+    const baseBlocks = parseSFC(source);
+    const blocks = framework === "svelte" ? [...baseBlocks, ...buildSvelteTemplateBlocks(source, baseBlocks)] : baseBlocks;
+    if (!blocks.length) {
+        return chunkTextFile({
+            source,
+            filePath,
+            languageId: framework,
+            minTokens,
+            maxTokens,
+            tokenizer,
+        });
+    }
+    const sortedBlocks = blocks.sort((a, b) => a.startOffset - b.startOffset || b.endOffset - a.endOffset);
+    const chunks = [];
+    for (const block of sortedBlocks) {
+        const blockChunks = chunkBlock({
+            block,
+            framework,
+            filePath,
+            minTokens,
+            maxTokens,
+            tokenizer,
+            ...(logLevel ? { logLevel } : {}),
+        });
+        for (const chunk of blockChunks) {
+            chunks.push({
+                ...chunk,
+                id: "",
+                startLine: chunk.startLine + block.startLine - 1,
+                endLine: chunk.endLine + block.startLine - 1,
+                type: `${block.type}:${chunk.type}`,
+                filePath: filePath ?? chunk.filePath,
+            });
+        }
+    }
+    // Semantic and text block chunkers cover every byte of non-blank block content.
+    // Keep each block's wrappers separate from inter-block text so inserting a
+    // preceding block cannot change an unchanged block's content-addressed misc IDs.
+    let coveredUntil = 0;
+    for (const block of sortedBlocks) {
+        if (block.blockStart < coveredUntil)
+            continue;
+        appendMiscChunks(chunks, source, coveredUntil, block.blockStart, framework, filePath, minTokens, maxTokens, tokenizer);
+        if (block.content.trim()) {
+            appendMiscChunks(chunks, source, block.blockStart, block.startOffset, framework, filePath, minTokens, maxTokens, tokenizer);
+            appendMiscChunks(chunks, source, block.endOffset, block.blockEnd, framework, filePath, minTokens, maxTokens, tokenizer);
+        }
+        else {
+            appendMiscChunks(chunks, source, block.blockStart, block.blockEnd, framework, filePath, minTokens, maxTokens, tokenizer);
+        }
+        coveredUntil = block.blockEnd;
+    }
+    appendMiscChunks(chunks, source, coveredUntil, source.length, framework, filePath, minTokens, maxTokens, tokenizer);
+    const sortedChunks = chunks.sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine);
+    return withStableChunkIds(sortedChunks, framework, filePath);
+}
+function appendMiscChunks(chunks, source, startOffset, endOffset, framework, filePath, minTokens, maxTokens, tokenizer) {
+    if (endOffset <= startOffset)
+        return;
+    const startLine = lineForOffset(source, startOffset);
+    const gapChunks = chunkTextFile({
+        source: source.slice(startOffset, endOffset),
+        filePath,
+        languageId: framework,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+    for (const chunk of gapChunks) {
+        chunks.push({
+            ...chunk,
+            id: "",
+            startLine: chunk.startLine + startLine - 1,
+            endLine: chunk.endLine + startLine - 1,
+            type: "misc",
+            filePath: filePath ?? chunk.filePath,
+        });
+    }
+}
+function lineForOffset(source, offset) {
+    let line = 1;
+    for (let index = 0; index < offset; index++) {
+        if (source[index] === "\n")
+            line++;
+    }
+    return line;
+}
+function chunkBlock(opts) {
+    const { block, framework, filePath, minTokens, maxTokens, tokenizer, logLevel } = opts;
+    if (!block.content.trim()) {
+        return [];
+    }
+    const languageKey = selectLanguageKey(block, framework);
+    if (languageKey) {
+        const config = LANG_CONFIGS[languageKey];
+        if (config) {
+            try {
+                return chunkFile({
+                    language: config,
+                    source: block.content,
+                    filePath,
+                    minTokens,
+                    maxTokens,
+                    tokenizer,
+                });
+            }
+            catch (error) {
+                logWithLevel(logLevel, "warn", `Warning: Semantic chunking failed for ${framework} ${block.type} block:`, error);
+            }
+        }
+    }
+    return chunkTextBlock(block, {
+        framework,
+        filePath,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+}
+function selectLanguageKey(block, framework) {
+    if (block.type === "script") {
+        const langId = scriptLanguageIdForBlock(block);
+        if (langId === "ts")
+            return "typescript";
+        if (langId === "tsx")
+            return "tsx";
+        return "javascript";
+    }
+    if (block.type === "style") {
+        const styleKey = styleLanguageKey(block);
+        if (!styleKey)
+            return null;
+        return styleKey;
+    }
+    if (block.type === "template") {
+        return templateLanguageKey(framework);
+    }
+    return null;
+}
+function chunkTextBlock(block, opts) {
+    const { framework, filePath, minTokens, maxTokens, tokenizer } = opts;
+    const languageId = `${framework}-${block.type}`;
+    const textChunks = chunkTextFile({
+        source: block.content,
+        filePath,
+        languageId,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+    return textChunks;
+}
+//# sourceMappingURL=chunkSFC.js.map

package/dist/chunking/chunkSFC.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSFC.js","sourceRoot":"","sources":["../../src/chunking/chunkSFC.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,qCAAqC,CAAC;AACnE,OAAO,EACL,QAAQ,EACR,yBAAyB,EACzB,wBAAwB,EACxB,gBAAgB,EAChB,mBAAmB,GAGpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAiB,MAAM,eAAe,CAAC;AAY5D;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAqB;IAChD,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACpG,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,MAAM,GACV,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAE1G,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,aAAa,CAAC;YACnB,MAAM;YACN,QAAQ;YACR,UAAU,EAAE,SAAS;YACrB,SAAS;YACT,SAAS;YACT,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IACvG,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,UAAU,CAAC;YAC7B,KAAK;YACL,SAAS;YACT,QAAQ;YACR,SAAS;YACT,SAAS;YACT,SAAS;YACT,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClC,CAAC,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,KAAK;gBACR,EAAE,EAAE,EAAE;gBACN,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC;gBAChD,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC;gBAC5C,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE;gBACnC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ;aACrC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,2EAA2E;IAC3E,iFAAiF;IACjF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,UAAU,GAAG,YAAY;YAAE,SAAS;QAC9C,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,YAAY,EACZ,KAAK,CAAC,UAAU,EAChB,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;QACF,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACzB,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,WAAW,EACjB,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;YACF,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,QAAQ,EACd,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gBAAgB,CACd,MAAM,EACN,MAAM,EACN,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,QAAQ,EACd,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,CACV,CAAC;QACJ,CAAC;QACD,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC;IAChC,CAAC;IACD,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEpH,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IACpH,OAAO,kBAAkB,CAAC,YAAY,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,gBAAgB,CACvB,MAAe,EACf,MAAc,EACd,WAAmB,EACnB,SAAiB,EACjB,SAAuB,EACvB,QAA4B,EAC5B,SAAiB,EACjB,SAAiB,EACjB,SAAiD;IAEjD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO;IACrC,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,aAAa,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;QAC5C,QAAQ;QACR,UAAU,EAAE,SAAS;QACrB,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC,CAAC;IACH,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,KAAK;YACR,EAAE,EAAE,EAAE;YACN,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC;YAC1C,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,SAAS,GAAG,CAAC;YACtC,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ;SACrC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc,EAAE,MAAc;IACnD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,IAAI,EAAE,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,IAQnB;IACC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAEvF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACxD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;QACzC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,OAAO,SAAS,CAAC;oBACf,QAAQ,EAAE,MAAM;oBAChB,MAAM,EAAE,KAAK,CAAC,OAAO;oBACrB,QAAQ;oBACR,SAAS;oBACT,SAAS;oBACT,SAAS;iBACV,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,CACV,QAAQ,EACR,MAAM,EACN,yCAAyC,SAAS,IAAI,KAAK,CAAC,IAAI,SAAS,EACzE,KAAK,CACN,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC,KAAK,EAAE;QAC3B,SAAS;QACT,QAAQ;QACR,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAe,EAAE,SAAuB;IACjE,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,YAAY,CAAC;QACzC,IAAI,MAAM,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QACnC,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CACrB,KAAe,EACf,IAMC;IAED,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACtE,MAAM,UAAU,GAAG,GAAG,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,UAAU,GAAG,aAAa,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,OAAO;QACrB,QAAQ;QACR,UAAU;QACV,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC,CAAC;IACH,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/chunking/chunkSplit.d.ts

@@ -0,0 +1,6 @@
+import type { BlockCandidate, ChunkTokenizer, RangedChunk } from "./types.js";
+export declare function splitLargeBlockSimple(block: BlockCandidate, source: string, tokenizer: ChunkTokenizer, maxTokens: number, out: RangedChunk[], languageId: string, filePath?: string): void;
+export declare function splitLargeBlockUsingInnerBlocks(block: BlockCandidate, _innerBlocks: BlockCandidate[], source: string, tokenizer: ChunkTokenizer, maxTokens: number, out: RangedChunk[], languageId: string, _newlineOffsets: number[], filePath?: string): void;
+/** Splits an oversized string on code-point boundaries using the configured tokenizer. */
+export declare function splitTextWithinTokenBudget(text: string, tokenizer: ChunkTokenizer, maxTokens: number): string[];
+//# sourceMappingURL=chunkSplit.d.ts.map

package/dist/chunking/chunkSplit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkSplit.d.ts","sourceRoot":"","sources":["../../src/chunking/chunkSplit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9E,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,EAAE,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,IAAI,CA2DN;AAED,wBAAgB,+BAA+B,CAC7C,KAAK,EAAE,cAAc,EACrB,YAAY,EAAE,cAAc,EAAE,EAC9B,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,EAAE,EAClB,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,MAAM,EAAE,EACzB,QAAQ,CAAC,EAAE,MAAM,GAChB,IAAI,CAMN;AAED,0FAA0F;AAC1F,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CA8C/G"}