@lzehrung/codegraph 2.3.20 → 2.3.22

package/dist/chunking/chunk-file.js

@@ -0,0 +1,152 @@
+import { collectChunkBlockGroups } from "./chunk-blocks.js";
+import { getChunkMatches } from "./chunk-matches.js";
+import { withStableChunkIds } from "./chunk-id.js";
+import { fillGapsWithMiscChunks, mergeSmallChunks } from "./chunk-merge.js";
+import { splitLargeBlockSimple, splitLargeBlockUsingInnerBlocks } from "./chunk-split.js";
+import { countWhitespaceTokens } from "./tokenizer.js";
+export function chunkFileWithSymbols(opts) {
+    const { language, source, filePath, minTokens = 150, maxTokens = 400, tokenizer = countWhitespaceTokens } = opts;
+    const matches = getChunkMatches(language, source, filePath);
+    const newlineOffsets = collectNewlineOffsets(source);
+    const { mainBlocks, innerBlocks, comments } = collectChunkBlockGroups(language, matches);
+    const selectedBlocks = selectMainBlocks(mainBlocks);
+    const preliminaryChunks = [];
+    const sortedInnerBlocks = [...innerBlocks].sort((left, right) => left.startByte - right.startByte || left.endByte - right.endByte);
+    for (const block of selectedBlocks) {
+        appendBlockChunks(block, sortedInnerBlocks, source, tokenizer, maxTokens, preliminaryChunks, language.id, newlineOffsets, filePath);
+    }
+    for (const comment of selectStandaloneComments(comments, selectedBlocks)) {
+        appendBlockChunks(comment, [], source, tokenizer, maxTokens, preliminaryChunks, language.id, newlineOffsets, filePath);
+    }
+    preliminaryChunks.sort((left, right) => left.sourceStart - right.sourceStart || right.sourceEnd - left.sourceEnd);
+    if (minTokens <= 1) {
+        const completeChunks = fillGapsWithMiscChunks(preliminaryChunks, source, language.id, filePath, tokenizer, 1, maxTokens, newlineOffsets);
+        const chunks = withStableChunkIds(completeChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), language.id, filePath);
+        return { chunks, symbolChunks: chunks };
+    }
+    const completeSymbolChunks = fillGapsWithMiscChunks(preliminaryChunks, source, language.id, filePath, tokenizer, 1, maxTokens, newlineOffsets);
+    const symbolChunks = withStableChunkIds(completeSymbolChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), language.id, filePath);
+    const mergedChunks = mergeSmallChunks([...preliminaryChunks], minTokens, maxTokens, tokenizer);
+    const completeChunks = fillGapsWithMiscChunks(mergedChunks, source, language.id, filePath, tokenizer, minTokens, maxTokens, newlineOffsets);
+    const chunks = withStableChunkIds(completeChunks.map(({ sourceStart: _sourceStart, sourceEnd: _sourceEnd, ...chunk }) => chunk), language.id, filePath);
+    return { chunks, symbolChunks };
+}
+export function chunkFile(opts) {
+    return chunkFileWithSymbols(opts).chunks;
+}
+function appendBlockChunks(block, innerBlocks, source, tokenizer, maxTokens, out, languageId, newlineOffsets, filePath) {
+    const text = source.slice(block.startByte, block.endByte);
+    const tokens = tokenizer(text);
+    if (tokens <= maxTokens) {
+        out.push({
+            id: "",
+            languageId,
+            type: block.kind,
+            startLine: block.startLine,
+            endLine: block.endLine,
+            text,
+            tokenCount: tokens,
+            sourceStart: block.startByte,
+            sourceEnd: block.endByte,
+            ...(filePath !== undefined ? { filePath } : {}),
+            ...(block.name !== undefined ? { name: block.name } : {}),
+        });
+        return;
+    }
+    const innerInRange = collectContainedInnerBlocks(innerBlocks, block);
+    if (!innerInRange.length) {
+        splitLargeBlockSimple(block, source, tokenizer, maxTokens, out, languageId, filePath);
+        return;
+    }
+    splitLargeBlockUsingInnerBlocks(block, innerInRange, source, tokenizer, maxTokens, out, languageId, newlineOffsets, filePath);
+}
+function collectContainedInnerBlocks(sortedInnerBlocks, block) {
+    let low = 0;
+    let high = sortedInnerBlocks.length;
+    while (low < high) {
+        const mid = (low + high) >> 1;
+        if (sortedInnerBlocks[mid].startByte <= block.startByte)
+            low = mid + 1;
+        else
+            high = mid;
+    }
+    const innerInRange = [];
+    for (let index = low; index < sortedInnerBlocks.length; index++) {
+        const innerBlock = sortedInnerBlocks[index];
+        if (innerBlock.startByte >= block.endByte)
+            break;
+        if (innerBlock.endByte < block.endByte)
+            innerInRange.push(innerBlock);
+    }
+    return innerInRange;
+}
+function selectMainBlocks(mainBlocks) {
+    const nodes = buildBlockTree(mainBlocks);
+    const selected = [];
+    const visit = (node) => {
+        selected.push(node.block);
+        for (const child of node.children) {
+            visit(child);
+        }
+    };
+    for (const node of nodes) {
+        visit(node);
+    }
+    return selected;
+}
+function buildBlockTree(mainBlocks) {
+    const sortedBlocks = deduplicateBlocks(mainBlocks).sort((left, right) => left.startByte - right.startByte || right.endByte - left.endByte);
+    const roots = [];
+    const stack = [];
+    for (const block of sortedBlocks) {
+        while (stack.length && stack[stack.length - 1].block.endByte <= block.startByte) {
+            stack.pop();
+        }
+        const parent = stack[stack.length - 1];
+        const node = { block, children: [] };
+        if (parent && block.endByte <= parent.block.endByte) {
+            parent.children.push(node);
+            stack.push(node);
+            continue;
+        }
+        if (parent && block.startByte < parent.block.endByte) {
+            continue;
+        }
+        roots.push(node);
+        stack.push(node);
+    }
+    return roots;
+}
+function deduplicateBlocks(blocks) {
+    const byRange = new Map();
+    for (const block of blocks) {
+        const key = `${block.startByte}:${block.endByte}`;
+        const existing = byRange.get(key);
+        if (!existing || (!existing.name && block.name) || (existing.name === block.name && block.kind < existing.kind)) {
+            byRange.set(key, block);
+        }
+    }
+    return [...byRange.values()];
+}
+function selectStandaloneComments(comments, selectedBlocks) {
+    const selectedComments = [];
+    for (const comment of deduplicateBlocks(comments)) {
+        if (selectedBlocks.some((block) => block.startByte <= comment.startByte && comment.endByte <= block.endByte)) {
+            continue;
+        }
+        const previous = selectedComments[selectedComments.length - 1];
+        if (previous && comment.startByte < previous.endByte)
+            continue;
+        selectedComments.push(comment);
+    }
+    return selectedComments;
+}
+function collectNewlineOffsets(source) {
+    const newlineOffsets = [];
+    for (let index = 0; index < source.length; index++) {
+        if (source[index] === "\n")
+            newlineOffsets.push(index);
+    }
+    return newlineOffsets;
+}
+//# sourceMappingURL=chunk-file.js.map

package/dist/chunking/chunk-file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-file.js","sourceRoot":"","sources":["../../src/chunking/chunk-file.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,qBAAqB,EAAE,+BAA+B,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAwCvD,MAAM,UAAU,oBAAoB,CAAC,IAAsB;IACzD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,GAAG,GAAG,EAAE,SAAS,GAAG,qBAAqB,EAAE,GAAG,IAAI,CAAC;IACjH,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5D,MAAM,cAAc,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,uBAAuB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzF,MAAM,cAAc,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAE5C,MAAM,iBAAiB,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,CAC7C,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAClF,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,iBAAiB,CACf,KAAK,EACL,iBAAiB,EACjB,MAAM,EACN,SAAS,EACT,SAAS,EACT,iBAAiB,EACjB,QAAQ,CAAC,EAAE,EACX,cAAc,EACd,QAAQ,CACT,CAAC;IACJ,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,wBAAwB,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,CAAC;QACzE,iBAAiB,CACf,OAAO,EACP,EAAE,EACF,MAAM,EACN,SAAS,EACT,SAAS,EACT,iBAAiB,EACjB,QAAQ,CAAC,EAAE,EACX,cAAc,EACd,QAAQ,CACT,CAAC;IACJ,CAAC;IAED,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAElH,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QACnB,MAAM,cAAc,GAAG,sBAAsB,CAC3C,iBAAiB,EACjB,MAAM,EACN,QAAQ,CAAC,EAAE,EACX,QAAQ,EACR,SAAS,EACT,CAAC,EACD,SAAS,EACT,cAAc,CACf,CAAC;QACF,MAAM,MAAM,GAAG,kBAAkB,CAC/B,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,EAC7F,QAAQ,CAAC,EAAE,EACX,QAAQ,CACT,CAAC;QACF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;IAC1C,CAAC;IAED,MAAM,oBAAoB,GAAG,sBAAsB,CACjD,iBAAiB,EACjB,MAAM,EACN,QAAQ,CAAC,EAAE,EACX,QAAQ,EACR,SAAS,EACT,CAAC,EACD,SAAS,EACT,cAAc,CACf,CAAC;IACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,EACnG,QAAQ,CAAC,EAAE,EACX,QAAQ,CACT,CAAC;IAEF,MAAM,YAAY,GAAG,gBAAgB,CAAC,CAAC,GAAG,iBAAiB,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAC/F,MAAM,cAAc,GAAG,sBAAsB,CAC3C,YAAY,EACZ,MAAM,EACN,QAAQ,CAAC,EAAE,EACX,QAAQ,EACR,SAAS,EACT,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;IACF,MAAM,MAAM,GAAG,kBAAkB,CAC/B,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,EAC7F,QAAQ,CAAC,EAAE,EACX,QAAQ,CACT,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsB;IAC9C,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAqB,EACrB,WAA6B,EAC7B,MAAc,EACd,SAAyB,EACzB,SAAiB,EACjB,GAAkB,EAClB,UAAkB,EAClB,cAAwB,EACxB,QAA4B;IAE5B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAE/B,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QACxB,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,EAAE;YACN,UAAU;YACV,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI;YACJ,UAAU,EAAE,MAAM;YAClB,WAAW,EAAE,KAAK,CAAC,SAAS;YAC5B,SAAS,EAAE,KAAK,CAAC,OAAO;YACxB,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1D,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,2BAA2B,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IACrE,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QACzB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACtF,OAAO;IACT,CAAC;IAED,+BAA+B,CAC7B,KAAK,EACL,YAAY,EACZ,MAAM,EACN,SAAS,EACT,SAAS,EACT,GAAG,EACH,UAAU,EACV,cAAc,EACd,QAAQ,CACT,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,iBAAmC,EAAE,KAAqB;IAC7F,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC;IACpC,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,iBAAiB,CAAC,GAAG,CAAE,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS;YAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;;YACnE,IAAI,GAAG,GAAG,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAqB,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,GAAG,EAAE,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAChE,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAE,CAAC;QAC7C,IAAI,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO;YAAE,MAAM;QACjD,IAAI,UAAU,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;YAAE,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,gBAAgB,CAAC,UAA4B;IACpD,MAAM,KAAK,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IAEtC,MAAM,KAAK,GAAG,CAAC,IAAe,EAAE,EAAE;QAChC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,KAAK,CAAC,KAAK,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,CAAC;IACd,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAOD,SAAS,cAAc,CAAC,UAA4B;IAClD,MAAM,YAAY,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC,IAAI,CACrD,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAClF,CAAC;IACF,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACjF,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,IAAI,GAAc,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAChD,IAAI,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QAED,IAAI,MAAM,IAAI,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACrD,SAAS;QACX,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAwB;IACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAElD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChH,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,wBAAwB,CAAC,QAA0B,EAAE,cAAgC;IAC5F,MAAM,gBAAgB,GAAqB,EAAE,CAAC;IAE9C,KAAK,MAAM,OAAO,IAAI,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClD,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7G,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,QAAQ,IAAI,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,OAAO;YAAE,SAAS;QAC/D,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAc;IAC3C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC"}

package/dist/chunking/chunk-id.d.ts

@@ -0,0 +1,7 @@
+import type { Chunk } from "./types.js";
+/**
+ * Assigns content-addressed IDs after chunking has finalized each chunk's text.
+ * Identical chunks use a deterministic suffix only to preserve uniqueness.
+ */
+export declare function withStableChunkIds(chunks: Chunk[], languageId: string, filePath: string | undefined): Chunk[];
+//# sourceMappingURL=chunk-id.d.ts.map

package/dist/chunking/chunk-id.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-id.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-id.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,EAAE,CA0B7G"}

package/dist/chunking/chunk-id.js

@@ -0,0 +1,30 @@
+import { createHash } from "node:crypto";
+/**
+ * Assigns content-addressed IDs after chunking has finalized each chunk's text.
+ * Identical chunks use a deterministic suffix only to preserve uniqueness.
+ */
+export function withStableChunkIds(chunks, languageId, filePath) {
+    const occurrences = new Map();
+    const scopedPath = filePath ?? "unknown";
+    return chunks.map((chunk) => {
+        const digest = createHash("sha256")
+            .update("codegraph-chunk-id-v1\0")
+            .update(languageId)
+            .update("\0")
+            .update(scopedPath)
+            .update("\0")
+            .update(chunk.type)
+            .update("\0")
+            .update(chunk.name ?? "")
+            .update("\0")
+            .update(chunk.text)
+            .digest("hex");
+        const occurrence = occurrences.get(digest) ?? 0;
+        occurrences.set(digest, occurrence + 1);
+        const id = occurrence === 0
+            ? `${languageId}:${scopedPath}:${digest}`
+            : `${languageId}:${scopedPath}:${digest}:${occurrence}`;
+        return { ...chunk, id };
+    });
+}
+//# sourceMappingURL=chunk-id.js.map

package/dist/chunking/chunk-id.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-id.js","sourceRoot":"","sources":["../../src/chunking/chunk-id.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe,EAAE,UAAkB,EAAE,QAA4B;IAClG,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,UAAU,GAAG,QAAQ,IAAI,SAAS,CAAC;IAEzC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;aAChC,MAAM,CAAC,yBAAyB,CAAC;aACjC,MAAM,CAAC,UAAU,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,UAAU,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;aACxB,MAAM,CAAC,IAAI,CAAC;aACZ,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,KAAK,CAAC,CAAC;QACjB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAChD,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,EAAE,GACN,UAAU,KAAK,CAAC;YACd,CAAC,CAAC,GAAG,UAAU,IAAI,UAAU,IAAI,MAAM,EAAE;YACzC,CAAC,CAAC,GAAG,UAAU,IAAI,UAAU,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;QAE5D,OAAO,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/chunking/chunk-matches.d.ts

@@ -0,0 +1,4 @@
+import type { LanguageConfig } from "./language-config.js";
+import type { ChunkMatch } from "./types.js";
+export declare function getChunkMatches(language: LanguageConfig, source: string, _filePath?: string | undefined): ChunkMatch[];
+//# sourceMappingURL=chunk-matches.d.ts.map

package/dist/chunking/chunk-matches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-matches.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-matches.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,cAAc,EACxB,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAC7B,UAAU,EAAE,CAYd"}

package/dist/chunking/chunk-matches.js

@@ -0,0 +1,65 @@
+import { Buffer } from "node:buffer";
+import { supportById } from "../languages.js";
+import { getNativeSingleQueryExecution } from "../native/tree-sitter-native.js";
+export function getChunkMatches(language, source, _filePath) {
+    const support = supportById(language.supportId);
+    if (support) {
+        const nativeExecution = getNativeSingleQueryExecution(source, support, language.queryText);
+        if (nativeExecution.matches) {
+            // Native Tree-sitter indexes are UTF-8 bytes; JS String.slice uses UTF-16 units.
+            const toStringIndex = createUtf8ByteToStringIndex(source);
+            return nativeExecution.matches.map((match) => toChunkMatchFromNative(match, toStringIndex));
+        }
+    }
+    return [];
+}
+function toChunkMatchFromNative(match, toStringIndex) {
+    return {
+        captures: match.captures.map((capture) => ({
+            name: capture.name,
+            text: capture.text,
+            startByte: toStringIndex(capture.start.index),
+            endByte: toStringIndex(capture.end.index),
+            startLine: capture.start.row + 1,
+            endLine: capture.end.row + 1,
+            nodeType: capture.nodeType,
+        })),
+    };
+}
+function createUtf8ByteToStringIndex(source) {
+    const utf8Length = Buffer.byteLength(source, "utf8");
+    if (utf8Length === source.length) {
+        return (byteOffset) => Math.max(0, Math.min(byteOffset, source.length));
+    }
+    const byteToStringIndex = new Uint32Array(utf8Length + 1);
+    let byteOffset = 0;
+    let stringIndex = 0;
+    while (stringIndex < source.length) {
+        const codePoint = source.codePointAt(stringIndex);
+        if (codePoint === undefined)
+            break;
+        const charStringLength = codePoint > 0xffff ? 2 : 1;
+        const charByteLength = utf8ByteLengthForCodePoint(codePoint);
+        for (let offset = 1; offset < charByteLength; offset += 1) {
+            byteToStringIndex[byteOffset + offset] = stringIndex;
+        }
+        byteOffset += charByteLength;
+        stringIndex += charStringLength;
+        byteToStringIndex[byteOffset] = stringIndex;
+    }
+    byteToStringIndex[byteOffset] = source.length;
+    return (index) => {
+        const bounded = Math.max(0, Math.min(index, byteToStringIndex.length - 1));
+        return byteToStringIndex[bounded] ?? source.length;
+    };
+}
+function utf8ByteLengthForCodePoint(codePoint) {
+    if (codePoint <= 0x7f)
+        return 1;
+    if (codePoint <= 0x7ff)
+        return 2;
+    if (codePoint <= 0xffff)
+        return 3;
+    return 4;
+}
+//# sourceMappingURL=chunk-matches.js.map

package/dist/chunking/chunk-matches.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-matches.js","sourceRoot":"","sources":["../../src/chunking/chunk-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,6BAA6B,EAAoB,MAAM,iCAAiC,CAAC;AAGlG,MAAM,UAAU,eAAe,CAC7B,QAAwB,EACxB,MAAc,EACd,SAA8B;IAE9B,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,eAAe,GAAG,6BAA6B,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC3F,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YAC5B,iFAAiF;YACjF,MAAM,aAAa,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;YAC1D,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAkB,EAAE,aAA6C;IAC/F,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACzC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,SAAS,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7C,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;YACzC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;YAChC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,MAAc;IACjD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,IAAI,UAAU,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACjC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAC1D,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,SAAS,KAAK,SAAS;YAAE,MAAM;QAEnC,MAAM,gBAAgB,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,0BAA0B,CAAC,SAAS,CAAC,CAAC;QAE7D,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YAC1D,iBAAiB,CAAC,UAAU,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC;QACvD,CAAC;QAED,UAAU,IAAI,cAAc,CAAC;QAC7B,WAAW,IAAI,gBAAgB,CAAC;QAChC,iBAAiB,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;IAC9C,CAAC;IAED,iBAAiB,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAE9C,OAAO,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,iBAAiB,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;IACrD,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,0BAA0B,CAAC,SAAiB;IACnD,IAAI,SAAS,IAAI,IAAI;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,SAAS,IAAI,KAAK;QAAE,OAAO,CAAC,CAAC;IACjC,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/chunking/chunk-merge.d.ts

@@ -0,0 +1,4 @@
+import type { ChunkTokenizer, RangedChunk } from "./types.js";
+export declare function mergeSmallChunks(chunks: RangedChunk[], minTokens: number, maxTokens: number, tokenizer: ChunkTokenizer): RangedChunk[];
+export declare function fillGapsWithMiscChunks(chunks: RangedChunk[], source: string, languageId: string, filePath: string | undefined, tokenizer: ChunkTokenizer, minTokens: number, maxTokens: number, newlineOffsets: number[]): RangedChunk[];
+//# sourceMappingURL=chunk-merge.d.ts.map

package/dist/chunking/chunk-merge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-merge.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-merge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAkB,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9E,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,WAAW,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,cAAc,GACxB,WAAW,EAAE,CAkDf;AAED,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,WAAW,EAAE,EACrB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,MAAM,EAAE,GACvB,WAAW,EAAE,CAmEf"}

package/dist/chunking/chunk-merge.js

@@ -0,0 +1,144 @@
+import { splitLargeBlockSimple } from "./chunk-split.js";
+export function mergeSmallChunks(chunks, minTokens, maxTokens, tokenizer) {
+    if (!chunks.length)
+        return [];
+    const merged = [];
+    let index = 0;
+    while (index < chunks.length) {
+        let current = { ...chunks[index] };
+        index++;
+        while (current.tokenCount < minTokens && index < chunks.length) {
+            if (current.type === "misc" && current.tokenCount === 0)
+                break;
+            const next = chunks[index];
+            // Hierarchical chunks may overlap, but nested ranges must remain separate.
+            // Only genuinely adjacent ranges can be concatenated without duplicating
+            // source text inside the resulting chunk.
+            if (next.sourceStart < current.sourceEnd)
+                break;
+            if (next.sourceStart !== current.sourceEnd)
+                break;
+            const combinedText = `${current.text}${next.text}`;
+            const combinedTokens = tokenizer(combinedText);
+            if (combinedTokens > maxTokens)
+                break;
+            const resolvedName = current.name ?? next.name;
+            let type = current.type;
+            if (current.type === "misc" && current.tokenCount === 0) {
+                type = next.type;
+            }
+            else if (next.type === "misc" && next.tokenCount === 0) {
+                type = current.type;
+            }
+            else if (current.type !== next.type) {
+                type = `${current.type}+${next.type}`;
+            }
+            current = {
+                ...current,
+                endLine: next.endLine,
+                sourceEnd: next.sourceEnd,
+                text: combinedText,
+                tokenCount: combinedTokens,
+                type,
+                ...(resolvedName !== undefined ? { name: resolvedName } : {}),
+            };
+            index++;
+        }
+        merged.push(current);
+    }
+    return merged;
+}
+export function fillGapsWithMiscChunks(chunks, source, languageId, filePath, tokenizer, minTokens, maxTokens, newlineOffsets) {
+    if (!source.length)
+        return [];
+    const result = [];
+    let currentOffset = 0;
+    const sortedChunks = [...chunks].sort((left, right) => left.sourceStart - right.sourceStart || right.sourceEnd - left.sourceEnd);
+    for (const chunk of sortedChunks) {
+        if (chunk.sourceStart > currentOffset) {
+            const appendedToPrevious = appendTriviaGap(result, source, currentOffset, chunk.sourceStart, tokenizer, maxTokens, newlineOffsets);
+            if (!appendedToPrevious) {
+                pushMiscRange(result, source, currentOffset, chunk.sourceStart, languageId, filePath, tokenizer, maxTokens, newlineOffsets);
+            }
+        }
+        // Keep nested declarations as independent chunks. They already cover bytes
+        // inside the parent, so they must not cause another misc chunk to be emitted.
+        result.push(chunk);
+        currentOffset = Math.max(currentOffset, chunk.sourceEnd);
+    }
+    if (currentOffset < source.length) {
+        const appendedToPrevious = appendTriviaGap(result, source, currentOffset, source.length, tokenizer, maxTokens, newlineOffsets);
+        if (!appendedToPrevious) {
+            pushMiscRange(result, source, currentOffset, source.length, languageId, filePath, tokenizer, maxTokens, newlineOffsets);
+        }
+    }
+    return mergeSmallChunks(result, minTokens, maxTokens, tokenizer);
+}
+function appendTriviaGap(chunks, source, start, end, tokenizer, maxTokens, newlineOffsets) {
+    if (!chunks.length || end <= start)
+        return false;
+    const previous = chunks[chunks.length - 1];
+    if (previous.sourceEnd !== start)
+        return false;
+    const text = source.slice(start, end);
+    if (!/^[\s;]+$/u.test(text))
+        return false;
+    const combinedText = `${previous.text}${text}`;
+    const tokenCount = tokenizer(combinedText);
+    if (tokenCount > maxTokens)
+        return false;
+    const [endRow] = locateLineFromOffset(newlineOffsets, Math.max(start, end - 1));
+    chunks[chunks.length - 1] = {
+        ...previous,
+        endLine: endRow + 1,
+        sourceEnd: end,
+        text: combinedText,
+        tokenCount,
+    };
+    return true;
+}
+function pushMiscRange(out, source, start, end, languageId, filePath, tokenizer, maxTokens, newlineOffsets) {
+    if (end <= start)
+        return;
+    const [startRow] = locateLineFromOffset(newlineOffsets, start);
+    const [endRow] = locateLineFromOffset(newlineOffsets, Math.max(start, end - 1));
+    const block = {
+        kind: "misc",
+        startByte: start,
+        endByte: end,
+        startLine: startRow + 1,
+        endLine: endRow + 1,
+    };
+    const text = source.slice(start, end);
+    const tokenCount = tokenizer(text);
+    if (tokenCount <= maxTokens) {
+        out.push({
+            id: "",
+            languageId,
+            type: "misc",
+            startLine: block.startLine,
+            endLine: block.endLine,
+            text,
+            tokenCount,
+            sourceStart: start,
+            sourceEnd: end,
+            ...(filePath !== undefined ? { filePath } : {}),
+        });
+        return;
+    }
+    splitLargeBlockSimple(block, source, tokenizer, maxTokens, out, languageId, filePath);
+}
+function locateLineFromOffset(newlineOffsets, offset) {
+    let low = 0;
+    let high = newlineOffsets.length;
+    while (low < high) {
+        const mid = (low + high) >>> 1;
+        if (newlineOffsets[mid] < offset)
+            low = mid + 1;
+        else
+            high = mid;
+    }
+    const previousNewline = low > 0 ? newlineOffsets[low - 1] : -1;
+    return [low, offset - previousNewline - 1];
+}
+//# sourceMappingURL=chunk-merge.js.map

package/dist/chunking/chunk-merge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-merge.js","sourceRoot":"","sources":["../../src/chunking/chunk-merge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,UAAU,gBAAgB,CAC9B,MAAqB,EACrB,SAAiB,EACjB,SAAiB,EACjB,SAAyB;IAEzB,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,OAAO,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAE,EAAE,CAAC;QACpC,KAAK,EAAE,CAAC;QAER,OAAO,OAAO,CAAC,UAAU,GAAG,SAAS,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC/D,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,CAAC;gBAAE,MAAM;YAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;YAE5B,2EAA2E;YAC3E,yEAAyE;YACzE,0CAA0C;YAC1C,IAAI,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,SAAS;gBAAE,MAAM;YAChD,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,CAAC,SAAS;gBAAE,MAAM;YAClD,MAAM,YAAY,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACnD,MAAM,cAAc,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,IAAI,cAAc,GAAG,SAAS;gBAAE,MAAM;YAEtC,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;YAE/C,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YACxB,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBACxD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBACzD,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YACtB,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtC,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACxC,CAAC;YAED,OAAO,GAAG;gBACR,GAAG,OAAO;gBACV,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,cAAc;gBAC1B,IAAI;gBACJ,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9D,CAAC;YACF,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,MAAqB,EACrB,MAAc,EACd,UAAkB,EAClB,QAA4B,EAC5B,SAAyB,EACzB,SAAiB,EACjB,SAAiB,EACjB,cAAwB;IAExB,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CACnC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAC1F,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,WAAW,GAAG,aAAa,EAAE,CAAC;YACtC,MAAM,kBAAkB,GAAG,eAAe,CACxC,MAAM,EACN,MAAM,EACN,aAAa,EACb,KAAK,CAAC,WAAW,EACjB,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;YACF,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,aAAa,CACX,MAAM,EACN,MAAM,EACN,aAAa,EACb,KAAK,CAAC,WAAW,EACjB,UAAU,EACV,QAAQ,EACR,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;YACJ,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,kBAAkB,GAAG,eAAe,CACxC,MAAM,EACN,MAAM,EACN,aAAa,EACb,MAAM,CAAC,MAAM,EACb,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;QACF,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,aAAa,CACX,MAAM,EACN,MAAM,EACN,aAAa,EACb,MAAM,CAAC,MAAM,EACb,UAAU,EACV,QAAQ,EACR,SAAS,EACT,SAAS,EACT,cAAc,CACf,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,eAAe,CACtB,MAAqB,EACrB,MAAc,EACd,KAAa,EACb,GAAW,EACX,SAAyB,EACzB,SAAiB,EACjB,cAAwB;IAExB,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAEjD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IAC5C,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1C,MAAM,YAAY,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;IAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;IAC3C,IAAI,UAAU,GAAG,SAAS;QAAE,OAAO,KAAK,CAAC;IAEzC,MAAM,CAAC,MAAM,CAAC,GAAG,oBAAoB,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;QAC1B,GAAG,QAAQ;QACX,OAAO,EAAE,MAAM,GAAG,CAAC;QACnB,SAAS,EAAE,GAAG;QACd,IAAI,EAAE,YAAY;QAClB,UAAU;KACX,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,GAAkB,EAClB,MAAc,EACd,KAAa,EACb,GAAW,EACX,UAAkB,EAClB,QAA4B,EAC5B,SAAyB,EACzB,SAAiB,EACjB,cAAwB;IAExB,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO;IAEzB,MAAM,CAAC,QAAQ,CAAC,GAAG,oBAAoB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAC/D,MAAM,CAAC,MAAM,CAAC,GAAG,oBAAoB,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAmB;QAC5B,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,GAAG;QACZ,SAAS,EAAE,QAAQ,GAAG,CAAC;QACvB,OAAO,EAAE,MAAM,GAAG,CAAC;KACpB,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEnC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC;YACP,EAAE,EAAE,EAAE;YACN,UAAU;YACV,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI;YACJ,UAAU;YACV,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,GAAG;YACd,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,oBAAoB,CAAC,cAAwB,EAAE,MAAc;IACpE,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC;IACjC,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,cAAc,CAAC,GAAG,CAAE,GAAG,MAAM;YAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;;YAC5C,IAAI,GAAG,GAAG,CAAC;IAClB,CAAC;IACD,MAAM,eAAe,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC"}

package/dist/chunking/chunk-sfc.d.ts

@@ -0,0 +1,18 @@
+import type { Chunk } from "./chunk-file.js";
+import { type SFCFramework } from "../languages/sfc.js";
+import { type LogLevel } from "../logging.js";
+export interface ChunkSFCOptions {
+    source: string;
+    filePath?: string | undefined;
+    framework: SFCFramework;
+    minTokens?: number;
+    maxTokens?: number;
+    tokenizer?: ((text: string) => number) | undefined;
+    logLevel?: LogLevel;
+}
+/**
+ * Chunks an SFC while preserving whole-file coverage: every source byte belongs
+ * to at least one semantic block chunk or bounded misc chunk.
+ */
+export declare function chunkSFCFile(opts: ChunkSFCOptions): Chunk[];
+//# sourceMappingURL=chunk-sfc.d.ts.map

package/dist/chunking/chunk-sfc.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-sfc.d.ts","sourceRoot":"","sources":["../../src/chunking/chunk-sfc.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAK7C,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAgB,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,SAAS,EAAE,YAAY,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,eAAe,GAAG,KAAK,EAAE,CAqG3D"}

package/dist/chunking/chunk-sfc.js

@@ -0,0 +1,165 @@
+import { chunkFile } from "./chunk-file.js";
+import { chunkTextFile } from "./chunk-text-file.js";
+import { withStableChunkIds } from "./chunk-id.js";
+import { LANG_CONFIGS } from "../bootstrap/tree-sitter-languages.js";
+import { parseSFC, buildSvelteTemplateBlocks, scriptLanguageIdForBlock, styleLanguageKey, templateLanguageKey, } from "../languages/sfc.js";
+import { logWithLevel } from "../logging.js";
+/**
+ * Chunks an SFC while preserving whole-file coverage: every source byte belongs
+ * to at least one semantic block chunk or bounded misc chunk.
+ */
+export function chunkSFCFile(opts) {
+    const { source, filePath, framework, minTokens = 150, maxTokens = 400, tokenizer, logLevel } = opts;
+    const baseBlocks = parseSFC(source);
+    const blocks = framework === "svelte" ? [...baseBlocks, ...buildSvelteTemplateBlocks(source, baseBlocks)] : baseBlocks;
+    if (!blocks.length) {
+        return chunkTextFile({
+            source,
+            filePath,
+            languageId: framework,
+            minTokens,
+            maxTokens,
+            tokenizer,
+        });
+    }
+    const sortedBlocks = blocks.sort((a, b) => a.startOffset - b.startOffset || b.endOffset - a.endOffset);
+    const chunks = [];
+    for (const block of sortedBlocks) {
+        const blockChunks = chunkBlock({
+            block,
+            framework,
+            filePath,
+            minTokens,
+            maxTokens,
+            tokenizer,
+            ...(logLevel ? { logLevel } : {}),
+        });
+        for (const chunk of blockChunks) {
+            chunks.push({
+                ...chunk,
+                id: "",
+                startLine: chunk.startLine + block.startLine - 1,
+                endLine: chunk.endLine + block.startLine - 1,
+                type: `${block.type}:${chunk.type}`,
+                filePath: filePath ?? chunk.filePath,
+            });
+        }
+    }
+    // Semantic and text block chunkers cover every byte of non-blank block content.
+    // Keep each block's wrappers separate from inter-block text so inserting a
+    // preceding block cannot change an unchanged block's content-addressed misc IDs.
+    let coveredUntil = 0;
+    for (const block of sortedBlocks) {
+        if (block.blockStart < coveredUntil)
+            continue;
+        appendMiscChunks(chunks, source, coveredUntil, block.blockStart, framework, filePath, minTokens, maxTokens, tokenizer);
+        if (block.content.trim()) {
+            appendMiscChunks(chunks, source, block.blockStart, block.startOffset, framework, filePath, minTokens, maxTokens, tokenizer);
+            appendMiscChunks(chunks, source, block.endOffset, block.blockEnd, framework, filePath, minTokens, maxTokens, tokenizer);
+        }
+        else {
+            appendMiscChunks(chunks, source, block.blockStart, block.blockEnd, framework, filePath, minTokens, maxTokens, tokenizer);
+        }
+        coveredUntil = block.blockEnd;
+    }
+    appendMiscChunks(chunks, source, coveredUntil, source.length, framework, filePath, minTokens, maxTokens, tokenizer);
+    const sortedChunks = chunks.sort((left, right) => left.startLine - right.startLine || left.endLine - right.endLine);
+    return withStableChunkIds(sortedChunks, framework, filePath);
+}
+function appendMiscChunks(chunks, source, startOffset, endOffset, framework, filePath, minTokens, maxTokens, tokenizer) {
+    if (endOffset <= startOffset)
+        return;
+    const startLine = lineForOffset(source, startOffset);
+    const gapChunks = chunkTextFile({
+        source: source.slice(startOffset, endOffset),
+        filePath,
+        languageId: framework,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+    for (const chunk of gapChunks) {
+        chunks.push({
+            ...chunk,
+            id: "",
+            startLine: chunk.startLine + startLine - 1,
+            endLine: chunk.endLine + startLine - 1,
+            type: "misc",
+            filePath: filePath ?? chunk.filePath,
+        });
+    }
+}
+function lineForOffset(source, offset) {
+    let line = 1;
+    for (let index = 0; index < offset; index++) {
+        if (source[index] === "\n")
+            line++;
+    }
+    return line;
+}
+function chunkBlock(opts) {
+    const { block, framework, filePath, minTokens, maxTokens, tokenizer, logLevel } = opts;
+    if (!block.content.trim()) {
+        return [];
+    }
+    const languageKey = selectLanguageKey(block, framework);
+    if (languageKey) {
+        const config = LANG_CONFIGS[languageKey];
+        if (config) {
+            try {
+                return chunkFile({
+                    language: config,
+                    source: block.content,
+                    filePath,
+                    minTokens,
+                    maxTokens,
+                    tokenizer,
+                });
+            }
+            catch (error) {
+                logWithLevel(logLevel, "warn", `Warning: Semantic chunking failed for ${framework} ${block.type} block:`, error);
+            }
+        }
+    }
+    return chunkTextBlock(block, {
+        framework,
+        filePath,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+}
+function selectLanguageKey(block, framework) {
+    if (block.type === "script") {
+        const langId = scriptLanguageIdForBlock(block);
+        if (langId === "ts")
+            return "typescript";
+        if (langId === "tsx")
+            return "tsx";
+        return "javascript";
+    }
+    if (block.type === "style") {
+        const styleKey = styleLanguageKey(block);
+        if (!styleKey)
+            return null;
+        return styleKey;
+    }
+    if (block.type === "template") {
+        return templateLanguageKey(framework);
+    }
+    return null;
+}
+function chunkTextBlock(block, opts) {
+    const { framework, filePath, minTokens, maxTokens, tokenizer } = opts;
+    const languageId = `${framework}-${block.type}`;
+    const textChunks = chunkTextFile({
+        source: block.content,
+        filePath,
+        languageId,
+        minTokens,
+        maxTokens,
+        tokenizer,
+    });
+    return textChunks;
+}
+//# sourceMappingURL=chunk-sfc.js.map