@zvec/zvec-grep 0.1.0

package/dist/engine/pipeline/indexing/extractor/code/languages/python.js

@@ -0,0 +1,53 @@
+import { extractCommonModifiers, extractGenericSignature, extractPrecedingDoc, } from "../families/metadata.js";
+export const PYTHON_ADAPTER = {
+    format: "python",
+    entityTypes: new Set([
+        "class_definition",
+        "decorated_definition",
+        "function_definition",
+    ]),
+    scopeTypes: new Set([
+        "class_definition",
+        "decorated_definition",
+    ]),
+    extractName(node) {
+        if (node.type === "decorated_definition") {
+            const inner = node.namedChildren.find((child) => child.type === "function_definition" || child.type === "class_definition");
+            return inner ? this.extractName(inner) : undefined;
+        }
+        return node.childForFieldName("name")?.text;
+    },
+    shouldEnterScope(node) {
+        if (node.type !== "decorated_definition") {
+            return true;
+        }
+        return node.namedChildren.some((child) => child.type === "class_definition");
+    },
+    enterScopeNode(node) {
+        if (node.type !== "decorated_definition") {
+            return node;
+        }
+        return node.namedChildren.find((child) => child.type === "class_definition") ?? node;
+    },
+    extractSignature(node) {
+        return extractGenericSignature(innerPythonDefinition(node) ?? node);
+    },
+    extractDoc: extractPrecedingDoc,
+    extractModifiers(node) {
+        const modifiers = new Set(extractCommonModifiers(node));
+        if (/^\s*async\s+def\b/m.test(node.text)) {
+            modifiers.add("async");
+        }
+        if (/^\s*@staticmethod\b/m.test(node.text)) {
+            modifiers.add("static");
+        }
+        return [...modifiers];
+    },
+};
+function innerPythonDefinition(node) {
+    if (node.type !== "decorated_definition") {
+        return undefined;
+    }
+    return node.namedChildren.find((child) => child.type === "function_definition" || child.type === "class_definition");
+}
+//# sourceMappingURL=python.js.map

package/dist/engine/pipeline/indexing/extractor/code/languages/python.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"python.js","sourceRoot":"","sources":["../../../../../../../src/engine/pipeline/indexing/extractor/code/languages/python.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,MAAM,CAAC,MAAM,cAAc,GAAoB;IAC7C,MAAM,EAAE,QAAQ;IAChB,WAAW,EAAE,IAAI,GAAG,CAAC;QACnB,kBAAkB;QAClB,sBAAsB;QACtB,qBAAqB;KACtB,CAAC;IACF,UAAU,EAAE,IAAI,GAAG,CAAC;QAClB,kBAAkB;QAClB,sBAAsB;KACvB,CAAC;IACF,WAAW,CAAC,IAAI;QACd,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAC9C,KAAK,CAAC,IAAI,KAAK,qBAAqB,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAC1E,CAAC;YACF,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;IAC9C,CAAC;IACD,gBAAgB,CAAC,IAAI;QACnB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC;IAC/E,CAAC;IACD,cAAc,CAAC,IAAI;QACjB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,IAAI,IAAI,CAAC;IACvF,CAAC;IACD,gBAAgB,CAAC,IAAI;QACnB,OAAO,uBAAuB,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACtE,CAAC;IACD,UAAU,EAAE,mBAAmB;IAC/B,gBAAgB,CAAC,IAAI;QACnB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAqB,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;QAE5E,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAED,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAED,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IACxB,CAAC;CACF,CAAC;AAGF,SAAS,qBAAqB,CAAC,IAAY;IACzC,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACvC,KAAK,CAAC,IAAI,KAAK,qBAAqB,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAC1E,CAAC;AACJ,CAAC"}

package/dist/engine/pipeline/indexing/extractor/code/languages/rust.d.ts

@@ -0,0 +1,2 @@
+import type { LanguageAdapter } from "../adapter.js";
+export declare const RUST_ADAPTER: LanguageAdapter;

package/dist/engine/pipeline/indexing/extractor/code/languages/rust.js

@@ -0,0 +1,29 @@
+import { extractCommonModifiers, extractGenericSignature, extractPrecedingDoc, } from "../families/metadata.js";
+export const RUST_ADAPTER = {
+    format: "rust",
+    entityTypes: new Set([
+        "enum_item",
+        "function_item",
+        "function_signature_item",
+        "impl_item",
+        "struct_item",
+        "trait_item",
+        "type_item",
+        "union_item",
+    ]),
+    scopeTypes: new Set([
+        "impl_item",
+        "mod_item",
+        "trait_item",
+    ]),
+    extractName(node) {
+        if (node.type === "impl_item") {
+            return node.childForFieldName("type")?.text;
+        }
+        return node.childForFieldName("name")?.text;
+    },
+    extractSignature: extractGenericSignature,
+    extractDoc: extractPrecedingDoc,
+    extractModifiers: extractCommonModifiers,
+};
+//# sourceMappingURL=rust.js.map

package/dist/engine/pipeline/indexing/extractor/code/languages/rust.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rust.js","sourceRoot":"","sources":["../../../../../../../src/engine/pipeline/indexing/extractor/code/languages/rust.ts"],"names":[],"mappings":"AACA,OAAO,EACL,sBAAsB,EACtB,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,MAAM,CAAC,MAAM,YAAY,GAAoB;IAC3C,MAAM,EAAE,MAAM;IACd,WAAW,EAAE,IAAI,GAAG,CAAC;QACnB,WAAW;QACX,eAAe;QACf,yBAAyB;QACzB,WAAW;QACX,aAAa;QACb,YAAY;QACZ,WAAW;QACX,YAAY;KACb,CAAC;IACF,UAAU,EAAE,IAAI,GAAG,CAAC;QAClB,WAAW;QACX,UAAU;QACV,YAAY;KACb,CAAC;IACF,WAAW,CAAC,IAAI;QACd,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;QAC9C,CAAC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;IAC9C,CAAC;IACD,gBAAgB,EAAE,uBAAuB;IACzC,UAAU,EAAE,mBAAmB;IAC/B,gBAAgB,EAAE,sBAAsB;CACzC,CAAC"}

package/dist/engine/pipeline/indexing/extractor/code/languages/typescript.d.ts

@@ -0,0 +1,2 @@
+import type { LanguageAdapter } from "../adapter.js";
+export declare const TYPESCRIPT_ADAPTER: LanguageAdapter;

package/dist/engine/pipeline/indexing/extractor/code/languages/typescript.js

@@ -0,0 +1,37 @@
+import { classifyJavascriptTypescriptNode, extractJavascriptTypescriptDoc, extractJavascriptTypescriptModifiers, extractJavascriptTypescriptName, extractJavascriptTypescriptSignature, javascriptTypescriptScopeBreadcrumb, resolveJavascriptTypescriptEntities, shouldIndexJavascriptTypescriptEntity, } from "../families/js-ts.js";
+export const TYPESCRIPT_ADAPTER = {
+    format: "typescript",
+    entityTypes: new Set([
+        "abstract_class_declaration",
+        "abstract_method_signature",
+        "class_declaration",
+        "enum_declaration",
+        "field_definition",
+        "function_declaration",
+        "generator_function_declaration",
+        "interface_declaration",
+        "method_signature",
+        "method_definition",
+        "pair",
+        "public_field_definition",
+        "type_alias_declaration",
+        "variable_declarator",
+    ]),
+    scopeTypes: new Set([
+        "abstract_class_declaration",
+        "class_declaration",
+        "internal_module",
+        "interface_declaration",
+        "module_declaration",
+        "namespace_declaration",
+    ]),
+    extractName: extractJavascriptTypescriptName,
+    shouldIndexEntity: shouldIndexJavascriptTypescriptEntity,
+    resolveEntities: resolveJavascriptTypescriptEntities,
+    scopeBreadcrumb: javascriptTypescriptScopeBreadcrumb,
+    classifyNode: classifyJavascriptTypescriptNode,
+    extractSignature: extractJavascriptTypescriptSignature,
+    extractDoc: extractJavascriptTypescriptDoc,
+    extractModifiers: extractJavascriptTypescriptModifiers,
+};
+//# sourceMappingURL=typescript.js.map

package/dist/engine/pipeline/indexing/extractor/code/languages/typescript.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"typescript.js","sourceRoot":"","sources":["../../../../../../../src/engine/pipeline/indexing/extractor/code/languages/typescript.ts"],"names":[],"mappings":"AACA,OAAO,EACL,gCAAgC,EAChC,8BAA8B,EAC9B,oCAAoC,EACpC,+BAA+B,EAC/B,oCAAoC,EACpC,mCAAmC,EACnC,mCAAmC,EACnC,qCAAqC,GACtC,MAAM,sBAAsB,CAAC;AAG9B,MAAM,CAAC,MAAM,kBAAkB,GAAoB;IACjD,MAAM,EAAE,YAAY;IACpB,WAAW,EAAE,IAAI,GAAG,CAAC;QACnB,4BAA4B;QAC5B,2BAA2B;QAC3B,mBAAmB;QACnB,kBAAkB;QAClB,kBAAkB;QAClB,sBAAsB;QACtB,gCAAgC;QAChC,uBAAuB;QACvB,kBAAkB;QAClB,mBAAmB;QACnB,MAAM;QACN,yBAAyB;QACzB,wBAAwB;QACxB,qBAAqB;KACtB,CAAC;IACF,UAAU,EAAE,IAAI,GAAG,CAAC;QAClB,4BAA4B;QAC5B,mBAAmB;QACnB,iBAAiB;QACjB,uBAAuB;QACvB,oBAAoB;QACpB,uBAAuB;KACxB,CAAC;IACF,WAAW,EAAE,+BAA+B;IAC5C,iBAAiB,EAAE,qCAAqC;IACxD,eAAe,EAAE,mCAAmC;IACpD,eAAe,EAAE,mCAAmC;IACpD,YAAY,EAAE,gCAAgC;IAC9C,gBAAgB,EAAE,oCAAoC;IACtD,UAAU,EAAE,8BAA8B;IAC1C,gBAAgB,EAAE,oCAAoC;CACvD,CAAC"}

package/dist/engine/pipeline/indexing/extractor/index.d.ts

@@ -0,0 +1,44 @@
+import type { EntityFragment, FileInfo, ImageFormat } from "../../../types.js";
+export type SourceKind = "text" | "image";
+type SourceBase = {
+    kind: SourceKind;
+    file: FileInfo;
+};
+export type TextSource = SourceBase & {
+    kind: "text";
+    text: string;
+};
+export type ImageSource = SourceBase & {
+    kind: "image";
+    data: Uint8Array;
+    format: ImageFormat;
+};
+export type Source = TextSource | ImageSource;
+export interface Extractor {
+    supports(source: Source): boolean;
+    extract(source: Source): Promise<EntityFragment[]>;
+}
+export type TextExtractorOptions = {
+    maxChunkChars?: number;
+    chunkOverlapChars?: number;
+};
+export declare class TextExtractor implements Extractor {
+    private readonly maxChunkChars;
+    private readonly chunkOverlapChars;
+    constructor(options?: TextExtractorOptions);
+    supports(source: Source): boolean;
+    extract(source: Source): Promise<EntityFragment[]>;
+}
+export declare class ImageExtractor implements Extractor {
+    supports(source: Source): boolean;
+    extract(source: Source): Promise<EntityFragment[]>;
+}
+export declare class ExtractorRegistry {
+    private readonly extractors;
+    constructor(extractors?: readonly Extractor[]);
+    resolve(source: Source): Extractor;
+    extract(source: Source): Promise<EntityFragment[]>;
+    private extractFirst;
+}
+export declare function extractFragments(source: Source): Promise<EntityFragment[]>;
+export {};

package/dist/engine/pipeline/indexing/extractor/index.js

@@ -0,0 +1,281 @@
+import { EngineError } from "../../../errors/index.js";
+import { makeEntityId } from "../ids.js";
+import { CodeExtractor } from "./code/extractor.js";
+import { MarkdownExtractor } from "./markdown/extractor.js";
+const DEFAULT_TEXT_CHUNK_CHARS = 3600;
+const DEFAULT_TEXT_CHUNK_OVERLAP_CHARS = 540;
+export class TextExtractor {
+    maxChunkChars;
+    chunkOverlapChars;
+    constructor(options = {}) {
+        const maxChunkChars = options.maxChunkChars ?? DEFAULT_TEXT_CHUNK_CHARS;
+        const chunkOverlapChars = options.chunkOverlapChars ?? DEFAULT_TEXT_CHUNK_OVERLAP_CHARS;
+        if (!Number.isInteger(maxChunkChars) || maxChunkChars <= 0) {
+            throw new EngineError("Text extractor requires a positive integer chunk size", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_INVALID_CHUNK_SIZE",
+                context: `maxChunkChars=${maxChunkChars}`,
+            });
+        }
+        if (!Number.isInteger(chunkOverlapChars)
+            || chunkOverlapChars < 0
+            || chunkOverlapChars >= maxChunkChars) {
+            throw new EngineError("Text extractor requires overlap to be smaller than chunk size", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_INVALID_CHUNK_OVERLAP",
+                context: `maxChunkChars=${maxChunkChars} chunkOverlapChars=${chunkOverlapChars}`,
+            });
+        }
+        this.maxChunkChars = maxChunkChars;
+        this.chunkOverlapChars = chunkOverlapChars;
+    }
+    supports(source) {
+        return source.kind === "text";
+    }
+    async extract(source) {
+        if (source.kind !== "text") {
+            throw new EngineError("Text extractor received a non-text source", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_UNSUPPORTED_SOURCE",
+                context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+            });
+        }
+        validateSourceFile(source);
+        const chunks = chunkText(source.text, this.maxChunkChars, this.chunkOverlapChars);
+        return chunks.map((chunk, index) => ({
+            id: makeEntityId(source.file.id, index),
+            fileId: source.file.id,
+            range: chunk.range,
+            content: {
+                kind: "text",
+                text: chunk.text,
+            },
+        }));
+    }
+}
+export class ImageExtractor {
+    supports(source) {
+        return source.kind === "image";
+    }
+    async extract(source) {
+        if (source.kind !== "image") {
+            throw new EngineError("Image extractor received a non-image source", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.IMAGE_UNSUPPORTED_SOURCE",
+                context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+            });
+        }
+        validateSourceFile(source);
+        if (source.data.byteLength === 0) {
+            throw new EngineError("Image extractor requires non-empty image data", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.IMAGE_EMPTY_DATA",
+                context: `fileId=${source.file.id} format=${source.format}`,
+            });
+        }
+        return [
+            {
+                id: makeEntityId(source.file.id, 0),
+                fileId: source.file.id,
+                range: {
+                    kind: "file",
+                },
+                content: {
+                    kind: "image",
+                    data: source.data,
+                    format: source.format,
+                },
+            },
+        ];
+    }
+}
+const DEFAULT_EXTRACTORS = [
+    new CodeExtractor(),
+    new MarkdownExtractor(),
+    new TextExtractor(),
+    new ImageExtractor(),
+];
+export class ExtractorRegistry {
+    extractors;
+    constructor(extractors = DEFAULT_EXTRACTORS) {
+        if (extractors.length === 0) {
+            throw new EngineError("Extractor registry requires at least one extractor", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_REGISTRY",
+            });
+        }
+        this.extractors = extractors;
+    }
+    resolve(source) {
+        for (const extractor of this.extractors) {
+            if (extractor.supports(source)) {
+                return extractor;
+            }
+        }
+        throw new EngineError("No extractor supports source", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.UNSUPPORTED_SOURCE",
+            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+        });
+    }
+    extract(source) {
+        return this.extractFirst(source);
+    }
+    async extractFirst(source) {
+        let matched = false;
+        let lastResult = [];
+        for (const extractor of this.extractors) {
+            if (!extractor.supports(source)) {
+                continue;
+            }
+            matched = true;
+            lastResult = await extractor.extract(source);
+            if (lastResult.length > 0) {
+                return lastResult;
+            }
+        }
+        if (matched) {
+            return lastResult;
+        }
+        throw new EngineError("No extractor supports source", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.UNSUPPORTED_SOURCE",
+            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+        });
+    }
+}
+export function extractFragments(source) {
+    return new ExtractorRegistry().extract(source);
+}
+function chunkText(text, maxChars, overlapChars) {
+    if (text.trim().length === 0) {
+        return [];
+    }
+    const lines = text.split("\n");
+    const lineOffsets = computeLineOffsets(lines);
+    const chunks = [];
+    let startIndex = 0;
+    while (startIndex < lines.length) {
+        if (lines[startIndex].length + 1 > maxChars) {
+            const line = lines[startIndex];
+            const lineNumber = startIndex + 1;
+            let offset = 0;
+            while (offset < line.length) {
+                const remaining = line.length - offset;
+                const sliceLength = remaining <= maxChars
+                    ? remaining
+                    : findLineCut(line.slice(offset), maxChars);
+                const slice = line.slice(offset, offset + sliceLength);
+                if (slice.trim().length > 0) {
+                    chunks.push({
+                        text: slice,
+                        range: {
+                            kind: "text",
+                            startLine: lineNumber,
+                            endLine: lineNumber,
+                            startOffset: lineOffsets[startIndex] + offset,
+                            endOffset: lineOffsets[startIndex] + offset + slice.length,
+                        },
+                    });
+                }
+                offset += sliceLength;
+            }
+            startIndex++;
+            continue;
+        }
+        let usedChars = 0;
+        let endIndex = startIndex;
+        while (endIndex < lines.length) {
+            const lineLength = lines[endIndex].length + 1;
+            if (usedChars + lineLength > maxChars && endIndex > startIndex) {
+                break;
+            }
+            usedChars += lineLength;
+            endIndex++;
+        }
+        const chunkLines = lines.slice(startIndex, endIndex);
+        const chunk = chunkLines.join("\n");
+        if (chunk.trim().length > 0) {
+            const endLineIndex = endIndex - 1;
+            chunks.push({
+                text: chunk,
+                range: {
+                    kind: "text",
+                    startLine: startIndex + 1,
+                    endLine: endIndex,
+                    startOffset: lineOffsets[startIndex],
+                    endOffset: lineOffsets[endLineIndex] + lines[endLineIndex].length,
+                },
+            });
+        }
+        if (endIndex >= lines.length) {
+            break;
+        }
+        startIndex = computeNextStartLine(lines, startIndex, endIndex, overlapChars);
+    }
+    return chunks;
+}
+function computeLineOffsets(lines) {
+    const offsets = [];
+    let offset = 0;
+    for (const line of lines) {
+        offsets.push(offset);
+        offset += line.length + 1;
+    }
+    return offsets;
+}
+function computeNextStartLine(lines, startIndex, endIndex, overlapChars) {
+    if (overlapChars === 0) {
+        return endIndex;
+    }
+    let overlapLines = 0;
+    let overlapCount = 0;
+    for (let index = endIndex - 1; index >= startIndex && overlapCount < overlapChars; index--) {
+        overlapCount += lines[index].length + 1;
+        overlapLines++;
+    }
+    const nextStart = endIndex - overlapLines;
+    return nextStart > startIndex ? nextStart : endIndex;
+}
+function findLineCut(line, maxChars) {
+    if (line.length <= maxChars) {
+        return line.length;
+    }
+    const minPosition = Math.floor(maxChars * 0.7);
+    let bestPosition = -1;
+    let bestScore = 0;
+    for (let index = minPosition; index < maxChars; index++) {
+        const character = line[index];
+        let score = 0;
+        if (character === "." || character === "!" || character === "?") {
+            score = 4;
+        }
+        else if (character === "," || character === ";" || character === ":") {
+            score = 3;
+        }
+        else if (character === " " || character === "\t") {
+            score = 2;
+        }
+        else if (character === "-" || character === "/" || character === "\\") {
+            score = 1;
+        }
+        if (score > 0 && score >= bestScore) {
+            bestScore = score;
+            bestPosition = index + 1;
+        }
+    }
+    return bestPosition > 0 ? bestPosition : maxChars;
+}
+function validateSourceFile(source) {
+    if (source.file.id.trim().length === 0) {
+        throw new EngineError("Extractor source requires a non-empty file id", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_FILE_ID",
+            context: `sourceKind=${source.kind}`,
+        });
+    }
+    if (source.file.absolutePath.trim().length === 0) {
+        throw new EngineError("Extractor source requires a non-empty absolute file path", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_ABSOLUTE_FILE_PATH",
+            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+        });
+    }
+    if (source.file.relativePath.trim().length === 0) {
+        throw new EngineError("Extractor source requires a non-empty relative file path", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_RELATIVE_FILE_PATH",
+            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+        });
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/engine/pipeline/indexing/extractor/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/engine/pipeline/indexing/extractor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAOvD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AA4C5D,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAGtC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAG7C,MAAM,OAAO,aAAa;IACP,aAAa,CAAS;IACtB,iBAAiB,CAAS;IAG3C,YAAY,UAAgC,EAAE;QAC5C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,wBAAwB,CAAC;QACxE,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,gCAAgC,CAAC;QAExF,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,WAAW,CAAC,uDAAuD,EAAE;gBAC7E,IAAI,EAAE,qDAAqD;gBAC3D,OAAO,EAAE,iBAAiB,aAAa,EAAE;aAC1C,CAAC,CAAC;QACL,CAAC;QAED,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC;eACjC,iBAAiB,GAAG,CAAC;eACrB,iBAAiB,IAAI,aAAa,EACrC,CAAC;YACD,MAAM,IAAI,WAAW,CAAC,+DAA+D,EAAE;gBACrF,IAAI,EAAE,wDAAwD;gBAC9D,OAAO,EAAE,iBAAiB,aAAa,sBAAsB,iBAAiB,EAAE;aACjF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7C,CAAC;IAGD,QAAQ,CAAC,MAAc;QACrB,OAAO,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC;IAChC,CAAC;IAGD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,WAAW,CAAC,2CAA2C,EAAE;gBACjE,IAAI,EAAE,qDAAqD;gBAC3D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;aAC9D,CAAC,CAAC;QACL,CAAC;QAED,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAE3B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAElF,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAkB,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;YACvC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE;gBACP,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,KAAK,CAAC,IAAI;aACjB;SACF,CAAC,CAAC,CAAC;IACN,CAAC;CACF;AAGD,MAAM,OAAO,cAAc;IACzB,QAAQ,CAAC,MAAc;QACrB,OAAO,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC;IACjC,CAAC;IAGD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,WAAW,CAAC,6CAA6C,EAAE;gBACnE,IAAI,EAAE,sDAAsD;gBAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;aAC9D,CAAC,CAAC;QACL,CAAC;QAED,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;gBACrE,IAAI,EAAE,8CAA8C;gBACpD,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,MAAM,CAAC,MAAM,EAAE;aAC5D,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL;gBACE,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;gBACtB,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM;iBACb;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;aACF;SACF,CAAC;IACJ,CAAC;CACF;AAGD,MAAM,kBAAkB,GAAyB;IAC/C,IAAI,aAAa,EAAE;IACnB,IAAI,iBAAiB,EAAE;IACvB,IAAI,aAAa,EAAE;IACnB,IAAI,cAAc,EAAE;CACrB,CAAC;AAGF,MAAM,OAAO,iBAAiB;IACX,UAAU,CAAuB;IAGlD,YAAY,aAAmC,kBAAkB;QAC/D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,WAAW,CAAC,oDAAoD,EAAE;gBAC1E,IAAI,EAAE,4CAA4C;aACnD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAGD,OAAO,CAAC,MAAc;QACpB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,WAAW,CAAC,8BAA8B,EAAE;YACpD,IAAI,EAAE,gDAAgD;YACtD,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CAAC,CAAC;IACL,CAAC;IAGD,OAAO,CAAC,MAAc;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAGO,KAAK,CAAC,YAAY,CAAC,MAAc;QACvC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,UAAU,GAAqB,EAAE,CAAC;QAEtC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,OAAO,GAAG,IAAI,CAAC;YACf,UAAU,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAE7C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,OAAO,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,MAAM,IAAI,WAAW,CAAC,8BAA8B,EAAE;YACpD,IAAI,EAAE,gDAAgD;YACtD,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CAAC,CAAC;IACL,CAAC;CACF;AAGD,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,OAAO,IAAI,iBAAiB,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AASD,SAAS,SAAS,CAAC,IAAY,EAAE,QAAgB,EAAE,YAAoB;IACrE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,EAAE,CAAC;YAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YAC/B,MAAM,UAAU,GAAG,UAAU,GAAG,CAAC,CAAC;YAClC,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvC,MAAM,WAAW,GAAG,SAAS,IAAI,QAAQ;oBACvC,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;gBAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;gBAEvD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE;4BACL,IAAI,EAAE,MAAM;4BACZ,SAAS,EAAE,UAAU;4BACrB,OAAO,EAAE,UAAU;4BACnB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM;4BAC7C,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM;yBAC3D;qBACF,CAAC,CAAC;gBACL,CAAC;gBAED,MAAM,IAAI,WAAW,CAAC;YACxB,CAAC;YAED,UAAU,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,QAAQ,GAAG,UAAU,CAAC;QAE1B,OAAO,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAE9C,IAAI,SAAS,GAAG,UAAU,GAAG,QAAQ,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC/D,MAAM;YACR,CAAC;YAED,SAAS,IAAI,UAAU,CAAC;YACxB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,QAAQ,GAAG,CAAC,CAAC;YAElC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,UAAU,GAAG,CAAC;oBACzB,OAAO,EAAE,QAAQ;oBACjB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC;oBACpC,SAAS,EAAE,WAAW,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,MAAM;iBAClE;aACF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM;QACR,CAAC;QAED,UAAU,GAAG,oBAAoB,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC/E,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAGD,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAGD,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,YAAoB;IAEpB,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,KAAK,IAAI,UAAU,IAAI,YAAY,GAAG,YAAY,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3F,YAAY,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC;IAE1C,OAAO,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvD,CAAC;AAGD,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAChE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC;AAGD,SAAS,kBAAkB,CAAC,MAAc;IACxC,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;YACrE,IAAI,EAAE,2CAA2C;YACjD,OAAO,EAAE,cAAc,MAAM,CAAC,IAAI,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CAAC,0DAA0D,EAAE;YAChF,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CAAC,0DAA0D,EAAE;YAChF,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}

package/dist/engine/pipeline/indexing/extractor/markdown/extractor.d.ts

@@ -0,0 +1,13 @@
+import type { EntityFragment } from "../../../../types.js";
+import type { Extractor, Source } from "../index.js";
+export type MarkdownExtractorOptions = {
+    maxChunkChars?: number;
+    chunkOverlapChars?: number;
+};
+export declare class MarkdownExtractor implements Extractor {
+    private readonly maxChunkChars;
+    private readonly chunkOverlapChars;
+    constructor(options?: MarkdownExtractorOptions);
+    supports(source: Source): boolean;
+    extract(source: Source): Promise<EntityFragment[]>;
+}