@zvec/zvec-grep 0.1.0

package/dist/engine/pipeline/indexing/extractor/markdown/extractor.js

@@ -0,0 +1,339 @@
+import { makeEntityId } from "../../ids.js";
+const DEFAULT_MARKDOWN_CHUNK_CHARS = 3600;
+const DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS = 540;
+export class MarkdownExtractor {
+    maxChunkChars;
+    chunkOverlapChars;
+    constructor(options = {}) {
+        this.maxChunkChars = options.maxChunkChars ?? DEFAULT_MARKDOWN_CHUNK_CHARS;
+        this.chunkOverlapChars = options.chunkOverlapChars ?? DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS;
+    }
+    supports(source) {
+        return source.kind === "text" && source.file.format === "markdown";
+    }
+    async extract(source) {
+        if (!this.supports(source) || source.kind !== "text") {
+            return [];
+        }
+        const lines = source.text.split("\n");
+        const headings = scanHeadings(lines);
+        if (headings.length === 0) {
+            return [];
+        }
+        const lineOffsets = computeLineOffsets(lines);
+        const fenceLines = computeFenceLines(lines);
+        const sections = buildSections(headings, lines);
+        const fragments = [];
+        for (const section of sections) {
+            const metadata = markdownMetadata(section);
+            const windows = splitMarkdownSection({
+                lines,
+                lineOffsets,
+                fenceLines,
+                section,
+                maxChars: this.maxChunkChars,
+                overlapChars: this.chunkOverlapChars,
+            });
+            if (windows.length > 1) {
+                const id = makeEntityId(source.file.id, fragments.length);
+                fragments.push({
+                    id,
+                    group: id,
+                    fileId: source.file.id,
+                    range: linesToWindow(lines, lineOffsets, section.startIndex, section.endIndex).range,
+                    content: {
+                        kind: "text",
+                        text: markdownOutline(metadata),
+                    },
+                    metadata,
+                });
+                for (const window of windows) {
+                    fragments.push(markdownWindowToFragment(source, metadata, window, fragments.length, id));
+                }
+                continue;
+            }
+            for (const window of windows) {
+                fragments.push({
+                    id: makeEntityId(source.file.id, fragments.length),
+                    fileId: source.file.id,
+                    range: window.range,
+                    content: {
+                        kind: "text",
+                        text: window.text,
+                    },
+                    metadata,
+                });
+            }
+        }
+        return fragments;
+    }
+}
+function markdownWindowToFragment(source, metadata, window, index, group) {
+    return {
+        id: makeEntityId(source.file.id, index),
+        group,
+        fileId: source.file.id,
+        range: window.range,
+        content: {
+            kind: "text",
+            text: window.text,
+        },
+        metadata,
+    };
+}
+function markdownOutline(metadata) {
+    return metadata.heading ?? "markdown section";
+}
+function scanHeadings(lines) {
+    const headings = [];
+    let fence = null;
+    for (let index = 0; index < lines.length; index++) {
+        const line = lines[index];
+        const trimmed = line.trimStart();
+        if (fence) {
+            if (trimmed.startsWith(fence)) {
+                fence = null;
+            }
+            continue;
+        }
+        if (trimmed.startsWith("```")) {
+            fence = "```";
+            continue;
+        }
+        if (trimmed.startsWith("~~~")) {
+            fence = "~~~";
+            continue;
+        }
+        const atx = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/);
+        if (atx) {
+            headings.push({
+                level: atx[1].length,
+                text: atx[2].trim(),
+                lineIndex: index,
+            });
+            continue;
+        }
+        const next = lines[index + 1]?.trim();
+        if (line.trim().length > 0 && next && /^(=+|-+)\s*$/.test(next)) {
+            headings.push({
+                level: next.startsWith("=") ? 1 : 2,
+                text: line.trim(),
+                lineIndex: index,
+            });
+            index++;
+        }
+    }
+    return headings;
+}
+function buildSections(headings, lines) {
+    const stack = [];
+    const sections = [];
+    const firstHeading = headings[0];
+    if (firstHeading.lineIndex > 0 && lines.slice(0, firstHeading.lineIndex).join("\n").trim().length > 0) {
+        sections.push({
+            heading: null,
+            startIndex: 0,
+            endIndex: firstHeading.lineIndex - 1,
+            breadcrumb: [],
+        });
+    }
+    for (let index = 0; index < headings.length; index++) {
+        const heading = headings[index];
+        while (stack.length > 0 && stack[stack.length - 1].level >= heading.level) {
+            stack.pop();
+        }
+        sections.push({
+            heading,
+            startIndex: heading.lineIndex,
+            endIndex: index + 1 < headings.length
+                ? headings[index + 1].lineIndex - 1
+                : lines.length - 1,
+            breadcrumb: stack.map((item) => item.text),
+        });
+        stack.push(heading);
+    }
+    return sections;
+}
+function splitMarkdownSection(input) {
+    const windows = [];
+    let startIndex = input.section.startIndex;
+    while (startIndex <= input.section.endIndex) {
+        if (input.lines[startIndex].length + 1 > input.maxChars) {
+            windows.push(...splitLongLine(input.lines[startIndex], startIndex, input.lineOffsets[startIndex], input.maxChars));
+            startIndex++;
+            continue;
+        }
+        let endIndex = startIndex;
+        let usedChars = 0;
+        while (endIndex <= input.section.endIndex) {
+            const lineLength = input.lines[endIndex].length + 1;
+            if (usedChars + lineLength > input.maxChars && endIndex > startIndex) {
+                break;
+            }
+            usedChars += lineLength;
+            endIndex++;
+        }
+        if (endIndex <= input.section.endIndex && endIndex - startIndex > 1) {
+            endIndex = chooseMarkdownBreak(input.lines, input.fenceLines, startIndex, endIndex);
+        }
+        windows.push(linesToWindow(input.lines, input.lineOffsets, startIndex, endIndex - 1));
+        if (endIndex > input.section.endIndex) {
+            break;
+        }
+        const overlapLines = computeMarkdownOverlapLines(input.lines, startIndex, endIndex, input.overlapChars);
+        const nextStart = endIndex - overlapLines;
+        startIndex = nextStart > startIndex ? nextStart : endIndex;
+    }
+    return windows.filter((window) => window.text.trim().length > 0);
+}
+function chooseMarkdownBreak(lines, fenceLines, startIndex, endIndex) {
+    const minBreak = startIndex + Math.max(1, Math.floor((endIndex - startIndex) * 0.7));
+    let bestBreak = endIndex;
+    let bestScore = markdownBreakScore(lines, fenceLines, endIndex);
+    for (let index = minBreak; index <= endIndex; index++) {
+        const score = markdownBreakScore(lines, fenceLines, index);
+        if (score > bestScore) {
+            bestBreak = index;
+            bestScore = score;
+        }
+    }
+    return bestBreak;
+}
+function markdownBreakScore(lines, fenceLines, breakIndex) {
+    if (breakIndex <= 0 || breakIndex >= lines.length || fenceLines[breakIndex]) {
+        return 0;
+    }
+    const current = lines[breakIndex].trim();
+    const previous = lines[breakIndex - 1].trim();
+    if (/^#{1,6}\s+/.test(current)) {
+        return 100;
+    }
+    if (previous === "" && current === "") {
+        return 70;
+    }
+    if (previous === "") {
+        return 60;
+    }
+    if (/^([-*+]|\d+\.)\s+/.test(current)) {
+        return 35;
+    }
+    if (/^>\s+/.test(current)) {
+        return 25;
+    }
+    return 10;
+}
+function splitLongLine(line, lineIndex, lineOffset, maxChars) {
+    const windows = [];
+    let offset = 0;
+    while (offset < line.length) {
+        const remaining = line.length - offset;
+        const sliceLength = remaining <= maxChars
+            ? remaining
+            : findLineCut(line.slice(offset), maxChars);
+        const text = line.slice(offset, offset + sliceLength);
+        windows.push({
+            text,
+            range: {
+                kind: "text",
+                startLine: lineIndex + 1,
+                endLine: lineIndex + 1,
+                startOffset: lineOffset + offset,
+                endOffset: lineOffset + offset + text.length,
+            },
+        });
+        offset += sliceLength;
+    }
+    return windows;
+}
+function linesToWindow(lines, lineOffsets, startIndex, endIndex) {
+    return {
+        text: lines.slice(startIndex, endIndex + 1).join("\n"),
+        range: {
+            kind: "text",
+            startLine: startIndex + 1,
+            endLine: endIndex + 1,
+            startOffset: lineOffsets[startIndex],
+            endOffset: lineOffsets[endIndex] + lines[endIndex].length,
+        },
+    };
+}
+function computeLineOffsets(lines) {
+    const offsets = [];
+    let offset = 0;
+    for (const line of lines) {
+        offsets.push(offset);
+        offset += line.length + 1;
+    }
+    return offsets;
+}
+function computeFenceLines(lines) {
+    const inFence = new Array(lines.length).fill(false);
+    let fence = null;
+    for (let index = 0; index < lines.length; index++) {
+        const trimmed = lines[index].trimStart();
+        if (fence) {
+            inFence[index] = true;
+            if (trimmed.startsWith(fence)) {
+                fence = null;
+            }
+            continue;
+        }
+        if (trimmed.startsWith("```")) {
+            fence = "```";
+        }
+        else if (trimmed.startsWith("~~~")) {
+            fence = "~~~";
+        }
+    }
+    return inFence;
+}
+function computeMarkdownOverlapLines(lines, startIndex, endIndex, overlapChars) {
+    if (overlapChars <= 0) {
+        return 0;
+    }
+    let chars = 0;
+    let count = 0;
+    for (let index = endIndex - 1; index > startIndex; index--) {
+        chars += lines[index].length + 1;
+        if (chars > overlapChars) {
+            break;
+        }
+        count++;
+    }
+    return Math.min(count, Math.floor((endIndex - startIndex) / 2));
+}
+function findLineCut(line, maxChars) {
+    const minPosition = Math.floor(maxChars * 0.7);
+    let bestPosition = -1;
+    let bestScore = 0;
+    for (let index = minPosition; index < maxChars; index++) {
+        const character = line[index];
+        let score = 0;
+        if (character === "." || character === "!" || character === "?") {
+            score = 4;
+        }
+        else if (character === "," || character === ";" || character === ":") {
+            score = 3;
+        }
+        else if (character === " " || character === "\t") {
+            score = 2;
+        }
+        else if (character === "-" || character === "/" || character === "\\") {
+            score = 1;
+        }
+        if (score >= bestScore && score > 0) {
+            bestScore = score;
+            bestPosition = index + 1;
+        }
+    }
+    return bestPosition > 0 ? bestPosition : maxChars;
+}
+function markdownMetadata(section) {
+    return {
+        kind: "markdown",
+        heading: section.heading?.text ?? null,
+        level: section.heading?.level ?? null,
+        scope: section.breadcrumb.length > 0 ? section.breadcrumb.join("::") : null,
+    };
+}
+//# sourceMappingURL=extractor.js.map

package/dist/engine/pipeline/indexing/extractor/markdown/extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../../../../src/engine/pipeline/indexing/extractor/markdown/extractor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAU5C,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAC1C,MAAM,oCAAoC,GAAG,GAAG,CAAC;AAwBjD,MAAM,OAAO,iBAAiB;IACX,aAAa,CAAS;IACtB,iBAAiB,CAAS;IAG3C,YAAY,UAAoC,EAAE;QAChD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,4BAA4B,CAAC;QAC3E,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,oCAAoC,CAAC;IAC7F,CAAC;IAGD,QAAQ,CAAC,MAAc;QACrB,OAAO,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;IACrE,CAAC;IAGD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,SAAS,GAAqB,EAAE,CAAC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,oBAAoB,CAAC;gBACnC,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,OAAO;gBACP,QAAQ,EAAE,IAAI,CAAC,aAAa;gBAC5B,YAAY,EAAE,IAAI,CAAC,iBAAiB;aACrC,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC1D,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE;oBACF,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK;oBACpF,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,eAAe,CAAC,QAAQ,CAAC;qBAChC;oBACD,QAAQ;iBACT,CAAC,CAAC;gBAEH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,SAAS,CAAC,IAAI,CAAC,wBAAwB,CACrC,MAAM,EACN,QAAQ,EACR,MAAM,EACN,SAAS,CAAC,MAAM,EAChB,EAAE,CACH,CAAC,CAAC;gBACL,CAAC;gBAED,SAAS;YACX,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;oBAClD,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB;oBACD,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAGD,SAAS,wBAAwB,CAC/B,MAAkB,EAClB,QAAgC,EAChC,MAAsB,EACtB,KAAa,EACb,KAAa;IAEb,OAAO;QACL,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;QACvC,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAGD,SAAS,eAAe,CAAC,QAAgC;IACvD,OAAO,QAAQ,CAAC,OAAO,IAAI,kBAAkB,CAAC;AAChD,CAAC;AAGD,SAAS,YAAY,CAAC,KAAwB;IAC5C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM;gBACpB,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACnB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;gBACjB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAGD,SAAS,aAAa,CAAC,QAA4B,EAAE,KAAwB;IAC3E,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEjC,IAAI,YAAY,CAAC,SAAS,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtG,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,YAAY,CAAC,SAAS,GAAG,CAAC;YACpC,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;IACL,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEhC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAC1E,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,QAAQ,EAAE,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACnC,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC;gBACnC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACpB,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3C,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAGD,SAAS,oBAAoB,CAAC,KAO7B;IACC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;IAE1C,OAAO,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAC3B,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EACvB,UAAU,EACV,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,EAC7B,KAAK,CAAC,QAAQ,CACf,CAAC,CAAC;YACH,UAAU,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,GAAG,UAAU,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACpD,IAAI,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBACrE,MAAM;YACR,CAAC;YACD,SAAS,IAAI,UAAU,CAAC;YACxB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,IAAI,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;YACpE,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;QAEtF,IAAI,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM;QACR,CAAC;QAED,MAAM,YAAY,GAAG,2BAA2B,CAC9C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,QAAQ,EACR,KAAK,CAAC,YAAY,CACnB,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC;QAC1C,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAGD,SAAS,mBAAmB,CAC1B,KAAwB,EACxB,UAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,MAAM,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACrF,IAAI,SAAS,GAAG,QAAQ,CAAC;IACzB,IAAI,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEhE,KAAK,IAAI,KAAK,GAAG,QAAQ,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAGD,SAAS,kBAAkB,CACzB,KAAwB,EACxB,UAA8B,EAC9B,UAAkB;IAElB,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAGD,SAAS,aAAa,CACpB,IAAY,EACZ,SAAiB,EACjB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvC,MAAM,WAAW,GAAG,SAAS,IAAI,QAAQ;YACvC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;QAEtD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI;YACJ,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,SAAS,GAAG,CAAC;gBACxB,OAAO,EAAE,SAAS,GAAG,CAAC;gBACtB,WAAW,EAAE,UAAU,GAAG,MAAM;gBAChC,SAAS,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM;aAC7C;SACF,CAAC,CAAC;QACH,MAAM,IAAI,WAAW,CAAC;IACxB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAGD,SAAS,aAAa,CACpB,KAAwB,EACxB,WAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACtD,KAAK,EAAE;YACL,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,UAAU,GAAG,CAAC;YACzB,OAAO,EAAE,QAAQ,GAAG,CAAC;YACrB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC;YACpC,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM;SAC1D;KACF,CAAC;AACJ,CAAC;AAGD,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAGD,SAAS,iBAAiB,CAAC,KAAwB;IACjD,MAAM,OAAO,GAAG,IAAI,KAAK,CAAU,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7D,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;QAEzC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACtB,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAGD,SAAS,2BAA2B,CAClC,KAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,YAAoB;IAEpB,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3D,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;YACzB,MAAM;QACR,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAGD,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAChE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QAED,IAAI,KAAK,IAAI,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC;AAGD,SAAS,gBAAgB,CAAC,OAAgB;IACxC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,IAAI,IAAI;QACtC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI;QACrC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAC5E,CAAC;AACJ,CAAC"}

package/dist/engine/pipeline/indexing/extractor/tree-sitter/grammar.d.ts

@@ -0,0 +1,4 @@
+import Parser from "web-tree-sitter";
+export declare function ensureParser(): Promise<void>;
+export declare function loadGrammar(format: string): Promise<Parser.Language | null>;
+export declare function hasGrammar(format: string): boolean;

package/dist/engine/pipeline/indexing/extractor/tree-sitter/grammar.js

@@ -0,0 +1,49 @@
+import { createRequire } from "node:module";
+import { join } from "node:path";
+import Parser from "web-tree-sitter";
+const require = createRequire(import.meta.url);
+const LANGUAGE_WASM_MAP = {
+    c: "c",
+    cpp: "cpp",
+    go: "go",
+    java: "java",
+    javascript: "javascript",
+    jsx: "javascript",
+    python: "python",
+    rust: "rust",
+    tsx: "tsx",
+    typescript: "typescript",
+};
+let parserReady = false;
+const grammarCache = new Map();
+export async function ensureParser() {
+    if (!parserReady) {
+        await Parser.init();
+        parserReady = true;
+    }
+}
+export async function loadGrammar(format) {
+    const cached = grammarCache.get(format);
+    if (cached) {
+        return cached;
+    }
+    const wasmName = LANGUAGE_WASM_MAP[format];
+    if (!wasmName) {
+        return null;
+    }
+    await ensureParser();
+    try {
+        const wasmsDir = join(require.resolve("tree-sitter-wasms/package.json"), "..", "out");
+        const wasmPath = join(wasmsDir, `tree-sitter-${wasmName}.wasm`);
+        const grammar = await Parser.Language.load(wasmPath);
+        grammarCache.set(format, grammar);
+        return grammar;
+    }
+    catch {
+        return null;
+    }
+}
+export function hasGrammar(format) {
+    return format in LANGUAGE_WASM_MAP;
+}
+//# sourceMappingURL=grammar.js.map

package/dist/engine/pipeline/indexing/extractor/tree-sitter/grammar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grammar.js","sourceRoot":"","sources":["../../../../../../src/engine/pipeline/indexing/extractor/tree-sitter/grammar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,MAAM,MAAM,iBAAiB,CAAC;AAGrC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAG/C,MAAM,iBAAiB,GAA2B;IAChD,CAAC,EAAE,GAAG;IACN,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,IAAI;IACR,IAAI,EAAE,MAAM;IACZ,UAAU,EAAE,YAAY;IACxB,GAAG,EAAE,YAAY;IACjB,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,UAAU,EAAE,YAAY;CACzB,CAAC;AAGF,IAAI,WAAW,GAAG,KAAK,CAAC;AACxB,MAAM,YAAY,GAAG,IAAI,GAAG,EAA2B,CAAC;AAGxD,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;AACH,CAAC;AAGD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc;IAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,YAAY,EAAE,CAAC;IAErB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,eAAe,QAAQ,OAAO,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAElC,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAGD,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,OAAO,MAAM,IAAI,iBAAiB,CAAC;AACrC,CAAC"}

package/dist/engine/pipeline/indexing/extractor/tree-sitter/nodes.d.ts

@@ -0,0 +1,3 @@
+import type Parser from "web-tree-sitter";
+export type TSNode = Parser.SyntaxNode;
+export declare function findIdentifierLeaf(node: TSNode): TSNode | null;

package/dist/engine/pipeline/indexing/extractor/tree-sitter/nodes.js

@@ -0,0 +1,26 @@
+export function findIdentifierLeaf(node) {
+    const wrappers = new Set([
+        "array_declarator",
+        "function_declarator",
+        "init_declarator",
+        "parenthesized_declarator",
+        "pointer_declarator",
+        "reference_declarator",
+    ]);
+    let current = node;
+    for (let depth = 0; current && depth < 16; depth++) {
+        if (current.type === "identifier" || current.type.endsWith("_identifier")) {
+            return current;
+        }
+        if (current.type === "destructor_name" || current.type === "operator_name") {
+            return current;
+        }
+        if (wrappers.has(current.type)) {
+            current = current.childForFieldName("declarator");
+            continue;
+        }
+        return null;
+    }
+    return null;
+}
+//# sourceMappingURL=nodes.js.map

package/dist/engine/pipeline/indexing/extractor/tree-sitter/nodes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nodes.js","sourceRoot":"","sources":["../../../../../../src/engine/pipeline/indexing/extractor/tree-sitter/nodes.ts"],"names":[],"mappings":"AAMA,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;QACvB,kBAAkB;QAClB,qBAAqB;QACrB,iBAAiB;QACjB,0BAA0B;QAC1B,oBAAoB;QACpB,sBAAsB;KACvB,CAAC,CAAC;IAEH,IAAI,OAAO,GAAkB,IAAI,CAAC;IAElC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,OAAO,IAAI,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1E,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,iBAAiB,IAAI,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;YAC3E,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/engine/pipeline/indexing/extractor/tree-sitter/parser.d.ts

@@ -0,0 +1,2 @@
+import Parser from "web-tree-sitter";
+export declare function withParser<T>(source: string, format: string, fn: (tree: Parser.Tree) => T | Promise<T>): Promise<T | null>;

package/dist/engine/pipeline/indexing/extractor/tree-sitter/parser.js

@@ -0,0 +1,31 @@
+import Parser from "web-tree-sitter";
+import { ensureParser, loadGrammar } from "./grammar.js";
+export async function withParser(source, format, fn) {
+    await ensureParser();
+    const grammar = await loadGrammar(format);
+    if (!grammar) {
+        return null;
+    }
+    const parser = new Parser();
+    parser.setLanguage(grammar);
+    let tree;
+    try {
+        tree = parser.parse(source);
+    }
+    catch {
+        parser.delete();
+        return null;
+    }
+    if (!tree) {
+        parser.delete();
+        return null;
+    }
+    try {
+        return await fn(tree);
+    }
+    finally {
+        tree.delete();
+        parser.delete();
+    }
+}
+//# sourceMappingURL=parser.js.map

package/dist/engine/pipeline/indexing/extractor/tree-sitter/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../../../../../src/engine/pipeline/indexing/extractor/tree-sitter/parser.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGzD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,MAAc,EACd,EAAyC;IAEzC,MAAM,YAAY,EAAE,CAAC;IAErB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE5B,IAAI,IAAwB,CAAC;IAE7B,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,EAAE,CAAC;IAClB,CAAC;AACH,CAAC"}

package/dist/engine/pipeline/indexing/ids.d.ts

@@ -0,0 +1 @@
+export declare function makeEntityId(fileId: string, index: number): string;

package/dist/engine/pipeline/indexing/ids.js

@@ -0,0 +1,5 @@
+import { sha256Text } from "../../utils/hash.js";
+export function makeEntityId(fileId, index) {
+    return sha256Text(`${fileId}\0${index}`);
+}
+//# sourceMappingURL=ids.js.map

package/dist/engine/pipeline/indexing/ids.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ids.js","sourceRoot":"","sources":["../../../../src/engine/pipeline/indexing/ids.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAGjD,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,KAAa;IACxD,OAAO,UAAU,CAAC,GAAG,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;AAC3C,CAAC"}

package/dist/engine/pipeline/indexing/index.d.ts

@@ -0,0 +1,12 @@
+import type { EmbeddingModel } from "../../models/embeddings.js";
+import type { CollectionStorage } from "../../storage/index.js";
+import type { CollectionIndexStatus, CollectionInfo, FileInfo, IndexProgress, IndexResult } from "../../types.js";
+export type IndexContext = {
+    collection: CollectionInfo;
+    storage: CollectionStorage;
+    embeddingModel: EmbeddingModel;
+    embeddingConcurrency?: number;
+    onProgress?: (progress: IndexProgress) => void;
+};
+export declare function indexCollection(ctx: IndexContext): Promise<IndexResult>;
+export declare function getCollectionIndexStatus(collection: CollectionInfo, storedFiles: readonly FileInfo[]): Promise<CollectionIndexStatus>;