@zvec/zvec-grep 0.2.2 → 0.3.0-rc.2

package/dist/engine/extraction/markdown/extractor.js

@@ -1,362 +0,0 @@
-import { EngineError } from "../../errors.js";
-import { makeEntityId } from "../ids.js";
-import { validateSourceFile } from "../source.js";
-import { extractPlainTextFragments } from "../text/extractor.js";
-import { chunkOptionsForMetadata, fitTextToChars } from "../vector-content.js";
-const DEFAULT_MARKDOWN_CHUNK_CHARS = 3600;
-const DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS = 540;
-export class MarkdownExtractor {
-    async extract(source, options = {}) {
-        if (source.kind !== "text" || source.file.format !== "markdown") {
-            return [];
-        }
-        validateSourceFile(source);
-        const chunkOptions = resolveMarkdownChunkOptions(options);
-        const lines = source.text.split("\n");
-        const headings = scanHeadings(lines);
-        if (headings.length === 0) {
-            return this.fallback(source, chunkOptions);
-        }
-        const lineOffsets = computeLineOffsets(lines);
-        const fenceLines = computeFenceLines(lines);
-        const sections = buildSections(headings, lines);
-        const fragments = [];
-        for (const section of sections) {
-            const metadata = markdownMetadata(section);
-            const contentChunkOptions = chunkOptionsForMetadata(chunkOptions, metadata);
-            const windows = splitMarkdownSection({
-                lines,
-                lineOffsets,
-                fenceLines,
-                section,
-                maxChars: contentChunkOptions.maxChunkChars,
-                overlapChars: contentChunkOptions.chunkOverlapChars,
-            });
-            if (windows.length > 1) {
-                const id = makeEntityId(source.file.id, fragments.length);
-                fragments.push({
-                    id,
-                    group: id,
-                    fileId: source.file.id,
-                    range: linesToWindow(lines, lineOffsets, section.startIndex, section.endIndex).range,
-                    content: {
-                        kind: "text",
-                        text: fitTextToChars(markdownOutline(metadata), contentChunkOptions.maxChunkChars),
-                    },
-                    metadata,
-                });
-                for (const window of windows) {
-                    fragments.push(markdownWindowToFragment(source, metadata, window, fragments.length, id));
-                }
-                continue;
-            }
-            for (const window of windows) {
-                fragments.push({
-                    id: makeEntityId(source.file.id, fragments.length),
-                    fileId: source.file.id,
-                    range: window.range,
-                    content: {
-                        kind: "text",
-                        text: window.text,
-                    },
-                    metadata,
-                });
-            }
-        }
-        return fragments.length > 0
-            ? fragments
-            : this.fallback(source, chunkOptions);
-    }
-    fallback(source, options) {
-        return extractPlainTextFragments(source, options.maxChunkChars, options.chunkOverlapChars);
-    }
-}
-function resolveMarkdownChunkOptions(options) {
-    const maxChunkChars = options.maxChunkChars ?? DEFAULT_MARKDOWN_CHUNK_CHARS;
-    const chunkOverlapChars = options.chunkOverlapChars ?? DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS;
-    if (!Number.isInteger(maxChunkChars) || maxChunkChars <= 0) {
-        throw new EngineError("Markdown extractor requires a positive integer chunk size", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.MARKDOWN_INVALID_CHUNK_SIZE",
-            context: `maxChunkChars=${maxChunkChars}`,
-        });
-    }
-    if (!Number.isInteger(chunkOverlapChars) ||
-        chunkOverlapChars < 0 ||
-        chunkOverlapChars >= maxChunkChars) {
-        throw new EngineError("Markdown extractor requires overlap to be smaller than chunk size", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.MARKDOWN_INVALID_CHUNK_OVERLAP",
-            context: `maxChunkChars=${maxChunkChars} chunkOverlapChars=${chunkOverlapChars}`,
-        });
-    }
-    return { maxChunkChars, chunkOverlapChars };
-}
-function markdownWindowToFragment(source, metadata, window, index, group) {
-    return {
-        id: makeEntityId(source.file.id, index),
-        group,
-        fileId: source.file.id,
-        range: window.range,
-        content: {
-            kind: "text",
-            text: window.text,
-        },
-        metadata,
-    };
-}
-function markdownOutline(metadata) {
-    return metadata.heading ?? "markdown section";
-}
-function scanHeadings(lines) {
-    const headings = [];
-    let fence = null;
-    for (let index = 0; index < lines.length; index++) {
-        const line = lines[index];
-        const trimmed = line.trimStart();
-        if (fence) {
-            if (trimmed.startsWith(fence)) {
-                fence = null;
-            }
-            continue;
-        }
-        if (trimmed.startsWith("```")) {
-            fence = "```";
-            continue;
-        }
-        if (trimmed.startsWith("~~~")) {
-            fence = "~~~";
-            continue;
-        }
-        const atx = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/);
-        if (atx) {
-            headings.push({
-                level: atx[1].length,
-                text: atx[2].trim(),
-                lineIndex: index,
-            });
-            continue;
-        }
-        const next = lines[index + 1]?.trim();
-        if (line.trim().length > 0 && next && /^(=+|-+)\s*$/.test(next)) {
-            headings.push({
-                level: next.startsWith("=") ? 1 : 2,
-                text: line.trim(),
-                lineIndex: index,
-            });
-            index++;
-        }
-    }
-    return headings;
-}
-function buildSections(headings, lines) {
-    const stack = [];
-    const sections = [];
-    const firstHeading = headings[0];
-    if (firstHeading.lineIndex > 0 &&
-        lines.slice(0, firstHeading.lineIndex).join("\n").trim().length > 0) {
-        sections.push({
-            heading: null,
-            startIndex: 0,
-            endIndex: firstHeading.lineIndex - 1,
-            breadcrumb: [],
-        });
-    }
-    for (let index = 0; index < headings.length; index++) {
-        const heading = headings[index];
-        while (stack.length > 0 && stack[stack.length - 1].level >= heading.level) {
-            stack.pop();
-        }
-        sections.push({
-            heading,
-            startIndex: heading.lineIndex,
-            endIndex: index + 1 < headings.length
-                ? headings[index + 1].lineIndex - 1
-                : lines.length - 1,
-            breadcrumb: stack.map((item) => item.text),
-        });
-        stack.push(heading);
-    }
-    return sections;
-}
-function splitMarkdownSection(input) {
-    const windows = [];
-    let startIndex = input.section.startIndex;
-    while (startIndex <= input.section.endIndex) {
-        if (input.lines[startIndex].length + 1 > input.maxChars) {
-            windows.push(...splitLongLine(input.lines[startIndex], startIndex, input.lineOffsets[startIndex], input.maxChars));
-            startIndex++;
-            continue;
-        }
-        let endIndex = startIndex;
-        let usedChars = 0;
-        while (endIndex <= input.section.endIndex) {
-            const lineLength = input.lines[endIndex].length + 1;
-            if (usedChars + lineLength > input.maxChars && endIndex > startIndex) {
-                break;
-            }
-            usedChars += lineLength;
-            endIndex++;
-        }
-        if (endIndex <= input.section.endIndex && endIndex - startIndex > 1) {
-            endIndex = chooseMarkdownBreak(input.lines, input.fenceLines, startIndex, endIndex);
-        }
-        windows.push(linesToWindow(input.lines, input.lineOffsets, startIndex, endIndex - 1));
-        if (endIndex > input.section.endIndex) {
-            break;
-        }
-        const overlapLines = computeMarkdownOverlapLines(input.lines, startIndex, endIndex, input.overlapChars);
-        const nextStart = endIndex - overlapLines;
-        startIndex = nextStart > startIndex ? nextStart : endIndex;
-    }
-    return windows.filter((window) => window.text.trim().length > 0);
-}
-function chooseMarkdownBreak(lines, fenceLines, startIndex, endIndex) {
-    const minBreak = startIndex + Math.max(1, Math.floor((endIndex - startIndex) * 0.7));
-    let bestBreak = endIndex;
-    let bestScore = markdownBreakScore(lines, fenceLines, endIndex);
-    for (let index = minBreak; index <= endIndex; index++) {
-        const score = markdownBreakScore(lines, fenceLines, index);
-        if (score > bestScore) {
-            bestBreak = index;
-            bestScore = score;
-        }
-    }
-    return bestBreak;
-}
-function markdownBreakScore(lines, fenceLines, breakIndex) {
-    if (breakIndex <= 0 || breakIndex >= lines.length || fenceLines[breakIndex]) {
-        return 0;
-    }
-    const current = lines[breakIndex].trim();
-    const previous = lines[breakIndex - 1].trim();
-    if (/^#{1,6}\s+/.test(current)) {
-        return 100;
-    }
-    if (previous === "" && current === "") {
-        return 70;
-    }
-    if (previous === "") {
-        return 60;
-    }
-    if (/^([-*+]|\d+\.)\s+/.test(current)) {
-        return 35;
-    }
-    if (/^>\s+/.test(current)) {
-        return 25;
-    }
-    return 10;
-}
-function splitLongLine(line, lineIndex, lineOffset, maxChars) {
-    const windows = [];
-    let offset = 0;
-    while (offset < line.length) {
-        const remaining = line.length - offset;
-        const sliceLength = remaining <= maxChars
-            ? remaining
-            : findLineCut(line.slice(offset), maxChars);
-        const text = line.slice(offset, offset + sliceLength);
-        windows.push({
-            text,
-            range: {
-                kind: "text",
-                startLine: lineIndex + 1,
-                endLine: lineIndex + 1,
-                startOffset: lineOffset + offset,
-                endOffset: lineOffset + offset + text.length,
-            },
-        });
-        offset += sliceLength;
-    }
-    return windows;
-}
-function linesToWindow(lines, lineOffsets, startIndex, endIndex) {
-    return {
-        text: lines.slice(startIndex, endIndex + 1).join("\n"),
-        range: {
-            kind: "text",
-            startLine: startIndex + 1,
-            endLine: endIndex + 1,
-            startOffset: lineOffsets[startIndex],
-            endOffset: lineOffsets[endIndex] + lines[endIndex].length,
-        },
-    };
-}
-function computeLineOffsets(lines) {
-    const offsets = [];
-    let offset = 0;
-    for (const line of lines) {
-        offsets.push(offset);
-        offset += line.length + 1;
-    }
-    return offsets;
-}
-function computeFenceLines(lines) {
-    const inFence = new Array(lines.length).fill(false);
-    let fence = null;
-    for (let index = 0; index < lines.length; index++) {
-        const trimmed = lines[index].trimStart();
-        if (fence) {
-            inFence[index] = true;
-            if (trimmed.startsWith(fence)) {
-                fence = null;
-            }
-            continue;
-        }
-        if (trimmed.startsWith("```")) {
-            fence = "```";
-        }
-        else if (trimmed.startsWith("~~~")) {
-            fence = "~~~";
-        }
-    }
-    return inFence;
-}
-function computeMarkdownOverlapLines(lines, startIndex, endIndex, overlapChars) {
-    if (overlapChars <= 0) {
-        return 0;
-    }
-    let chars = 0;
-    let count = 0;
-    for (let index = endIndex - 1; index > startIndex; index--) {
-        chars += lines[index].length + 1;
-        if (chars > overlapChars) {
-            break;
-        }
-        count++;
-    }
-    return Math.min(count, Math.floor((endIndex - startIndex) / 2));
-}
-function findLineCut(line, maxChars) {
-    const minPosition = Math.floor(maxChars * 0.7);
-    let bestPosition = -1;
-    let bestScore = 0;
-    for (let index = minPosition; index < maxChars; index++) {
-        const character = line[index];
-        let score = 0;
-        if (character === "." || character === "!" || character === "?") {
-            score = 4;
-        }
-        else if (character === "," || character === ";" || character === ":") {
-            score = 3;
-        }
-        else if (character === " " || character === "\t") {
-            score = 2;
-        }
-        else if (character === "-" || character === "/" || character === "\\") {
-            score = 1;
-        }
-        if (score >= bestScore && score > 0) {
-            bestScore = score;
-            bestPosition = index + 1;
-        }
-    }
-    return bestPosition > 0 ? bestPosition : maxChars;
-}
-function markdownMetadata(section) {
-    return {
-        kind: "markdown",
-        heading: section.heading?.text ?? null,
-        level: section.heading?.level ?? null,
-        scope: section.breadcrumb.length > 0 ? section.breadcrumb.join("::") : null,
-    };
-}
-//# sourceMappingURL=extractor.js.map

package/dist/engine/extraction/markdown/extractor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../../src/engine/extraction/markdown/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAM9C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAgC,MAAM,cAAc,CAAC;AAChF,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE/E,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAC1C,MAAM,oCAAoC,GAAG,GAAG,CAAC;AAoBjD,MAAM,OAAO,iBAAiB;IAC5B,KAAK,CAAC,OAAO,CACX,MAAc,EACd,UAAwB,EAAE;QAE1B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,YAAY,GAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE1D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,SAAS,GAAqB,EAAE,CAAC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,mBAAmB,GAAG,uBAAuB,CACjD,YAAY,EACZ,QAAQ,CACT,CAAC;YACF,MAAM,OAAO,GAAG,oBAAoB,CAAC;gBACnC,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,OAAO;gBACP,QAAQ,EAAE,mBAAmB,CAAC,aAAa;gBAC3C,YAAY,EAAE,mBAAmB,CAAC,iBAAiB;aACpD,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC1D,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE;oBACF,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,aAAa,CAClB,KAAK,EACL,WAAW,EACX,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,QAAQ,CACjB,CAAC,KAAK;oBACP,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,cAAc,CAClB,eAAe,CAAC,QAAQ,CAAC,EACzB,mBAAmB,CAAC,aAAa,CAClC;qBACF;oBACD,QAAQ;iBACT,CAAC,CAAC;gBAEH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,SAAS,CAAC,IAAI,CACZ,wBAAwB,CACtB,MAAM,EACN,QAAQ,EACR,MAAM,EACN,SAAS,CAAC,MAAM,EAChB,EAAE,CACH,CACF,CAAC;gBACJ,CAAC;gBAED,SAAS;YACX,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;oBAClD,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB;oBACD,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC1C,CAAC;IAEO,QAAQ,CACd,MAAkB,EAClB,OAA+B;QAE/B,OAAO,yBAAyB,CAC9B,MAAM,EACN,OAAO,CAAC,aAAa,EACrB,OAAO,CAAC,iBAAiB,CAC1B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,OAAqB;IAErB,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,4BAA4B,CAAC;IAC5E,MAAM,iBAAiB,GACrB,OAAO,CAAC,iBAAiB,IAAI,oCAAoC,CAAC;IAEpE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,WAAW,CACnB,2DAA2D,EAC3D;YACE,IAAI,EAAE,yDAAyD;YAC/D,OAAO,EAAE,iBAAiB,aAAa,EAAE;SAC1C,CACF,CAAC;IACJ,CAAC;IAED,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC;QACpC,iBAAiB,GAAG,CAAC;QACrB,iBAAiB,IAAI,aAAa,EAClC,CAAC;QACD,MAAM,IAAI,WAAW,CACnB,mEAAmE,EACnE;YACE,IAAI,EAAE,4DAA4D;YAClE,OAAO,EAAE,iBAAiB,aAAa,sBAAsB,iBAAiB,EAAE;SACjF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAAkB,EAClB,QAAgC,EAChC,MAAsB,EACtB,KAAa,EACb,KAAa;IAEb,OAAO;QACL,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;QACvC,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAgC;IACvD,OAAO,QAAQ,CAAC,OAAO,IAAI,kBAAkB,CAAC;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,KAAwB;IAC5C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM;gBACpB,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACnB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;gBACjB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B,EAC5B,KAAwB;IAExB,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEjC,IACE,YAAY,CAAC,SAAS,GAAG,CAAC;QAC1B,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EACnE,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,YAAY,CAAC,SAAS,GAAG,CAAC;YACpC,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;IACL,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEhC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAC1E,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,QAAQ,EACN,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACzB,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC;gBACnC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACtB,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3C,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,oBAAoB,CAAC,KAO7B;IACC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;IAE1C,OAAO,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YACxD,OAAO,CAAC,IAAI,CACV,GAAG,aAAa,CACd,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EACvB,UAAU,EACV,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,EAC7B,KAAK,CAAC,QAAQ,CACf,CACF,CAAC;YACF,UAAU,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,GAAG,UAAU,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACpD,IAAI,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBACrE,MAAM;YACR,CAAC;YACD,SAAS,IAAI,UAAU,CAAC;YACxB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,IAAI,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;YACpE,QAAQ,GAAG,mBAAmB,CAC5B,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,UAAU,EAChB,UAAU,EACV,QAAQ,CACT,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CACV,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CACxE,CAAC;QAEF,IAAI,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM;QACR,CAAC;QAED,MAAM,YAAY,GAAG,2BAA2B,CAC9C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,QAAQ,EACR,KAAK,CAAC,YAAY,CACnB,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC;QAC1C,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAwB,EACxB,UAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,MAAM,QAAQ,GACZ,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACtE,IAAI,SAAS,GAAG,QAAQ,CAAC;IACzB,IAAI,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEhE,KAAK,IAAI,KAAK,GAAG,QAAQ,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAwB,EACxB,UAA8B,EAC9B,UAAkB;IAElB,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,SAAiB,EACjB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvC,MAAM,WAAW,GACf,SAAS,IAAI,QAAQ;YACnB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;QAEtD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI;YACJ,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,SAAS,GAAG,CAAC;gBACxB,OAAO,EAAE,SAAS,GAAG,CAAC;gBACtB,WAAW,EAAE,UAAU,GAAG,MAAM;gBAChC,SAAS,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM;aAC7C;SACF,CAAC,CAAC;QACH,MAAM,IAAI,WAAW,CAAC;IACxB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,WAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACtD,KAAK,EAAE;YACL,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,UAAU,GAAG,CAAC;YACzB,OAAO,EAAE,QAAQ,GAAG,CAAC;YACrB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC;YACpC,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM;SAC1D;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAwB;IACjD,MAAM,OAAO,GAAG,IAAI,KAAK,CAAU,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7D,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;QAEzC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACtB,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,2BAA2B,CAClC,KAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,YAAoB;IAEpB,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3D,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;YACzB,MAAM;QACR,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAChE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QAED,IAAI,KAAK,IAAI,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAgB;IACxC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,IAAI,IAAI;QACtC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI;QACrC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAC5E,CAAC;AACJ,CAAC"}

package/dist/engine/extraction/runtime.d.ts

@@ -1,9 +0,0 @@
-import type { Content, EntityFragment } from "../types.js";
-import type { Source } from "./source.js";
-import type { ChunkOptions } from "./types.js";
-export declare function extract(source: Source, options?: ChunkOptions): Promise<EntityFragment[]>;
-export type IndexingExtractionFragment = {
-    fragment: EntityFragment;
-    embeddingSource?: Content;
-};
-export declare function extractForIndexing(source: Source, options?: ChunkOptions): Promise<IndexingExtractionFragment[]>;

package/dist/engine/extraction/runtime.js

@@ -1,37 +0,0 @@
-import { CodeExtractor } from "./code/extractor.js";
-import { ImageExtractor } from "./image/extractor.js";
-import { MarkdownExtractor } from "./markdown/extractor.js";
-import { TextExtractor } from "./text/extractor.js";
-const extractors = {
-    code: new CodeExtractor(),
-    image: new ImageExtractor(),
-    markdown: new MarkdownExtractor(),
-    text: new TextExtractor(),
-};
-export function extract(source, options = {}) {
-    return extractors[routeSource(source)].extract(source, options);
-}
-export async function extractForIndexing(source, options = {}) {
-    if (routeSource(source) === "code") {
-        return (await extractors.code.extractForIndexing(source, options)).map(({ fragment, embeddingText }) => ({
-            fragment,
-            ...(embeddingText === undefined
-                ? {}
-                : { embeddingSource: { kind: "text", text: embeddingText } }),
-        }));
-    }
-    return (await extractors[routeSource(source)].extract(source, options)).map((fragment) => ({ fragment }));
-}
-function routeSource(source) {
-    if (source.kind === "image") {
-        return "image";
-    }
-    if (source.file.kind === "code") {
-        return "code";
-    }
-    if (source.file.format === "markdown") {
-        return "markdown";
-    }
-    return "text";
-}
-//# sourceMappingURL=runtime.js.map

package/dist/engine/extraction/runtime.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../src/engine/extraction/runtime.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AASpD,MAAM,UAAU,GAAG;IACjB,IAAI,EAAE,IAAI,aAAa,EAAE;IACzB,KAAK,EAAE,IAAI,cAAc,EAAE;IAC3B,QAAQ,EAAE,IAAI,iBAAiB,EAAE;IACjC,IAAI,EAAE,IAAI,aAAa,EAAE;CACwB,CAAC;AAEpD,MAAM,UAAU,OAAO,CACrB,MAAc,EACd,UAAwB,EAAE;IAE1B,OAAO,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAClE,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,UAAwB,EAAE;IAE1B,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,MAAM,EAAE,CAAC;QACnC,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CACpE,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ;YACR,GAAG,CAAC,aAAa,KAAK,SAAS;gBAC7B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,CAAC;SAChE,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,MAAM,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CACzE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAAc;IACjC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACtC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/engine/extraction/source.d.ts

@@ -1,18 +0,0 @@
-import type { FileInfo, ImageFormat } from "../types.js";
-export type SourceKind = "text" | "image";
-type SourceBase = {
-    kind: SourceKind;
-    file: FileInfo;
-};
-export type TextSource = SourceBase & {
-    kind: "text";
-    text: string;
-};
-export type ImageSource = SourceBase & {
-    kind: "image";
-    data: Uint8Array;
-    format: ImageFormat;
-};
-export type Source = TextSource | ImageSource;
-export declare function validateSourceFile(source: Source): void;
-export {};

package/dist/engine/extraction/source.js

@@ -1,22 +0,0 @@
-import { EngineError } from "../errors.js";
-export function validateSourceFile(source) {
-    if (source.file.id.trim().length === 0) {
-        throw new EngineError("Extractor source requires a non-empty file id", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_FILE_ID",
-            context: `sourceKind=${source.kind}`,
-        });
-    }
-    if (source.file.absolutePath.trim().length === 0) {
-        throw new EngineError("Extractor source requires a non-empty absolute file path", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_ABSOLUTE_FILE_PATH",
-            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
-        });
-    }
-    if (source.file.relativePath.trim().length === 0) {
-        throw new EngineError("Extractor source requires a non-empty relative file path", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_RELATIVE_FILE_PATH",
-            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
-        });
-    }
-}
-//# sourceMappingURL=source.js.map

package/dist/engine/extraction/source.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"source.js","sourceRoot":"","sources":["../../../src/engine/extraction/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAuB3C,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;YACrE,IAAI,EAAE,2CAA2C;YACjD,OAAO,EAAE,cAAc,MAAM,CAAC,IAAI,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CACnB,0DAA0D,EAC1D;YACE,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CACnB,0DAA0D,EAC1D;YACE,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CACF,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/engine/extraction/text/extractor.d.ts

@@ -1,7 +0,0 @@
-import type { EntityFragment } from "../../types.js";
-import { type Source, type TextSource } from "../source.js";
-import type { ChunkOptions } from "../types.js";
-export declare class TextExtractor {
-    extract(source: Source, options?: ChunkOptions): Promise<EntityFragment[]>;
-}
-export declare function extractPlainTextFragments(source: TextSource, maxChunkChars: number, chunkOverlapChars: number): EntityFragment[];

package/dist/engine/extraction/text/extractor.js

@@ -1,168 +0,0 @@
-import { EngineError } from "../../errors.js";
-import { makeEntityId } from "../ids.js";
-import { validateSourceFile } from "../source.js";
-const DEFAULT_TEXT_CHUNK_CHARS = 3600;
-const DEFAULT_TEXT_CHUNK_OVERLAP_CHARS = 540;
-export class TextExtractor {
-    async extract(source, options = {}) {
-        if (source.kind !== "text") {
-            throw new EngineError("Text extractor received a non-text source", {
-                code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_UNSUPPORTED_SOURCE",
-                context: `fileId=${source.file.id} sourceKind=${source.kind}`,
-            });
-        }
-        validateSourceFile(source);
-        const { maxChunkChars, chunkOverlapChars } = resolveTextChunkOptions(options);
-        return extractPlainTextFragments(source, maxChunkChars, chunkOverlapChars);
-    }
-}
-function resolveTextChunkOptions(options) {
-    const maxChunkChars = options.maxChunkChars ?? DEFAULT_TEXT_CHUNK_CHARS;
-    const chunkOverlapChars = options.chunkOverlapChars ?? DEFAULT_TEXT_CHUNK_OVERLAP_CHARS;
-    if (!Number.isInteger(maxChunkChars) || maxChunkChars <= 0) {
-        throw new EngineError("Text extractor requires a positive integer chunk size", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_INVALID_CHUNK_SIZE",
-            context: `maxChunkChars=${maxChunkChars}`,
-        });
-    }
-    if (!Number.isInteger(chunkOverlapChars) ||
-        chunkOverlapChars < 0 ||
-        chunkOverlapChars >= maxChunkChars) {
-        throw new EngineError("Text extractor requires overlap to be smaller than chunk size", {
-            code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_INVALID_CHUNK_OVERLAP",
-            context: `maxChunkChars=${maxChunkChars} chunkOverlapChars=${chunkOverlapChars}`,
-        });
-    }
-    return { maxChunkChars, chunkOverlapChars };
-}
-export function extractPlainTextFragments(source, maxChunkChars, chunkOverlapChars) {
-    return chunkText(source.text, maxChunkChars, chunkOverlapChars).map((chunk, index) => ({
-        id: makeEntityId(source.file.id, index),
-        fileId: source.file.id,
-        range: chunk.range,
-        content: {
-            kind: "text",
-            text: chunk.text,
-        },
-    }));
-}
-function chunkText(text, maxChars, overlapChars) {
-    if (text.trim().length === 0) {
-        return [];
-    }
-    const lines = text.split("\n");
-    const lineOffsets = computeLineOffsets(lines);
-    const chunks = [];
-    let startIndex = 0;
-    while (startIndex < lines.length) {
-        if (lines[startIndex].length + 1 > maxChars) {
-            const line = lines[startIndex];
-            const lineNumber = startIndex + 1;
-            let offset = 0;
-            while (offset < line.length) {
-                const remaining = line.length - offset;
-                const sliceLength = remaining <= maxChars
-                    ? remaining
-                    : findLineCut(line.slice(offset), maxChars);
-                const slice = line.slice(offset, offset + sliceLength);
-                if (slice.trim().length > 0) {
-                    chunks.push({
-                        text: slice,
-                        range: {
-                            kind: "text",
-                            startLine: lineNumber,
-                            endLine: lineNumber,
-                            startOffset: lineOffsets[startIndex] + offset,
-                            endOffset: lineOffsets[startIndex] + offset + slice.length,
-                        },
-                    });
-                }
-                offset += sliceLength;
-            }
-            startIndex++;
-            continue;
-        }
-        let usedChars = 0;
-        let endIndex = startIndex;
-        while (endIndex < lines.length) {
-            const lineLength = lines[endIndex].length + 1;
-            if (usedChars + lineLength > maxChars && endIndex > startIndex) {
-                break;
-            }
-            usedChars += lineLength;
-            endIndex++;
-        }
-        const chunkLines = lines.slice(startIndex, endIndex);
-        const chunk = chunkLines.join("\n");
-        if (chunk.trim().length > 0) {
-            const endLineIndex = endIndex - 1;
-            chunks.push({
-                text: chunk,
-                range: {
-                    kind: "text",
-                    startLine: startIndex + 1,
-                    endLine: endIndex,
-                    startOffset: lineOffsets[startIndex],
-                    endOffset: lineOffsets[endLineIndex] + lines[endLineIndex].length,
-                },
-            });
-        }
-        if (endIndex >= lines.length) {
-            break;
-        }
-        startIndex = computeNextStartLine(lines, startIndex, endIndex, overlapChars);
-    }
-    return chunks;
-}
-function computeLineOffsets(lines) {
-    const offsets = [];
-    let offset = 0;
-    for (const line of lines) {
-        offsets.push(offset);
-        offset += line.length + 1;
-    }
-    return offsets;
-}
-function computeNextStartLine(lines, startIndex, endIndex, overlapChars) {
-    if (overlapChars === 0) {
-        return endIndex;
-    }
-    let overlapLines = 0;
-    let overlapCount = 0;
-    for (let index = endIndex - 1; index >= startIndex && overlapCount < overlapChars; index--) {
-        overlapCount += lines[index].length + 1;
-        overlapLines++;
-    }
-    const nextStart = endIndex - overlapLines;
-    return nextStart > startIndex ? nextStart : endIndex;
-}
-function findLineCut(line, maxChars) {
-    if (line.length <= maxChars) {
-        return line.length;
-    }
-    const minPosition = Math.floor(maxChars * 0.7);
-    let bestPosition = -1;
-    let bestScore = 0;
-    for (let index = minPosition; index < maxChars; index++) {
-        const character = line[index];
-        let score = 0;
-        if (character === "." || character === "!" || character === "?") {
-            score = 4;
-        }
-        else if (character === "," || character === ";" || character === ":") {
-            score = 3;
-        }
-        else if (character === " " || character === "\t") {
-            score = 2;
-        }
-        else if (character === "-" || character === "/" || character === "\\") {
-            score = 1;
-        }
-        if (score > 0 && score >= bestScore) {
-            bestScore = score;
-            bestPosition = index + 1;
-        }
-    }
-    return bestPosition > 0 ? bestPosition : maxChars;
-}
-//# sourceMappingURL=extractor.js.map