@zvec/zvec-grep 0.1.5 → 0.1.6

package/dist/engine/extraction/markdown/extractor.js

@@ -1,25 +1,20 @@
+import { EngineError } from "../../errors.js";
 import { makeEntityId } from "../ids.js";
+import { validateSourceFile } from "../source.js";
+import { extractPlainTextFragments } from "../text/extractor.js";
 const DEFAULT_MARKDOWN_CHUNK_CHARS = 3600;
 const DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS = 540;
 export class MarkdownExtractor {
-    maxChunkChars;
-    chunkOverlapChars;
-    constructor(options = {}) {
-        this.maxChunkChars = options.maxChunkChars ?? DEFAULT_MARKDOWN_CHUNK_CHARS;
-        this.chunkOverlapChars =
-            options.chunkOverlapChars ?? DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS;
-    }
-    supports(source) {
-        return source.kind === "text" && source.file.format === "markdown";
-    }
-    async extract(source) {
-        if (!this.supports(source) || source.kind !== "text") {
+    async extract(source, options = {}) {
+        if (source.kind !== "text" || source.file.format !== "markdown") {
             return [];
         }
+        validateSourceFile(source);
+        const chunkOptions = resolveMarkdownChunkOptions(options);
         const lines = source.text.split("\n");
         const headings = scanHeadings(lines);
         if (headings.length === 0) {
-            return [];
+            return this.fallback(source, chunkOptions);
         }
         const lineOffsets = computeLineOffsets(lines);
         const fenceLines = computeFenceLines(lines);
@@ -32,8 +27,8 @@ export class MarkdownExtractor {
                 lineOffsets,
                 fenceLines,
                 section,
-                maxChars: this.maxChunkChars,
-                overlapChars: this.chunkOverlapChars,
+                maxChars: chunkOptions.maxChunkChars,
+                overlapChars: chunkOptions.chunkOverlapChars,
             });
             if (windows.length > 1) {
                 const id = makeEntityId(source.file.id, fragments.length);
@@ -66,8 +61,32 @@ export class MarkdownExtractor {
                 });
             }
         }
-        return fragments;
+        return fragments.length > 0
+            ? fragments
+            : this.fallback(source, chunkOptions);
+    }
+    fallback(source, options) {
+        return extractPlainTextFragments(source, options.maxChunkChars, options.chunkOverlapChars);
+    }
+}
+function resolveMarkdownChunkOptions(options) {
+    const maxChunkChars = options.maxChunkChars ?? DEFAULT_MARKDOWN_CHUNK_CHARS;
+    const chunkOverlapChars = options.chunkOverlapChars ?? DEFAULT_MARKDOWN_CHUNK_OVERLAP_CHARS;
+    if (!Number.isInteger(maxChunkChars) || maxChunkChars <= 0) {
+        throw new EngineError("Markdown extractor requires a positive integer chunk size", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.MARKDOWN_INVALID_CHUNK_SIZE",
+            context: `maxChunkChars=${maxChunkChars}`,
+        });
+    }
+    if (!Number.isInteger(chunkOverlapChars) ||
+        chunkOverlapChars < 0 ||
+        chunkOverlapChars >= maxChunkChars) {
+        throw new EngineError("Markdown extractor requires overlap to be smaller than chunk size", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.MARKDOWN_INVALID_CHUNK_OVERLAP",
+            context: `maxChunkChars=${maxChunkChars} chunkOverlapChars=${chunkOverlapChars}`,
+        });
     }
+    return { maxChunkChars, chunkOverlapChars };
 }
 function markdownWindowToFragment(source, metadata, window, index, group) {
     return {

package/dist/engine/extraction/markdown/extractor.js.map

@@ -1 +1 @@
-{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../../src/engine/extraction/markdown/extractor.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAQzC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAC1C,MAAM,oCAAoC,GAAG,GAAG,CAAC;AAoBjD,MAAM,OAAO,iBAAiB;IACX,aAAa,CAAS;IACtB,iBAAiB,CAAS;IAE3C,YAAY,UAAoC,EAAE;QAChD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,4BAA4B,CAAC;QAC3E,IAAI,CAAC,iBAAiB;YACpB,OAAO,CAAC,iBAAiB,IAAI,oCAAoC,CAAC;IACtE,CAAC;IAED,QAAQ,CAAC,MAAc;QACrB,OAAO,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,SAAS,GAAqB,EAAE,CAAC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,oBAAoB,CAAC;gBACnC,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,OAAO;gBACP,QAAQ,EAAE,IAAI,CAAC,aAAa;gBAC5B,YAAY,EAAE,IAAI,CAAC,iBAAiB;aACrC,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC1D,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE;oBACF,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,aAAa,CAClB,KAAK,EACL,WAAW,EACX,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,QAAQ,CACjB,CAAC,KAAK;oBACP,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,eAAe,CAAC,QAAQ,CAAC;qBAChC;oBACD,QAAQ;iBACT,CAAC,CAAC;gBAEH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,SAAS,CAAC,IAAI,CACZ,wBAAwB,CACtB,MAAM,EACN,QAAQ,EACR,MAAM,EACN,SAAS,CAAC,MAAM,EAChB,EAAE,CACH,CACF,CAAC;gBACJ,CAAC;gBAED,SAAS;YACX,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;oBAClD,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB;oBACD,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAED,SAAS,wBAAwB,CAC/B,MAAkB,EAClB,QAAgC,EAChC,MAAsB,EACtB,KAAa,EACb,KAAa;IAEb,OAAO;QACL,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;QACvC,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAgC;IACvD,OAAO,QAAQ,CAAC,OAAO,IAAI,kBAAkB,CAAC;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,KAAwB;IAC5C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM;gBACpB,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACnB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;gBACjB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B,EAC5B,KAAwB;IAExB,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEjC,IACE,YAAY,CAAC,SAAS,GAAG,CAAC;QAC1B,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EACnE,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,YAAY,CAAC,SAAS,GAAG,CAAC;YACpC,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;IACL,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEhC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAC1E,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,QAAQ,EACN,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACzB,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC;gBACnC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACtB,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3C,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,oBAAoB,CAAC,KAO7B;IACC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;IAE1C,OAAO,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YACxD,OAAO,CAAC,IAAI,CACV,GAAG,aAAa,CACd,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EACvB,UAAU,EACV,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,EAC7B,KAAK,CAAC,QAAQ,CACf,CACF,CAAC;YACF,UAAU,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,GAAG,UAAU,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACpD,IAAI,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBACrE,MAAM;YACR,CAAC;YACD,SAAS,IAAI,UAAU,CAAC;YACxB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,IAAI,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;YACpE,QAAQ,GAAG,mBAAmB,CAC5B,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,UAAU,EAChB,UAAU,EACV,QAAQ,CACT,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CACV,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CACxE,CAAC;QAEF,IAAI,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM;QACR,CAAC;QAED,MAAM,YAAY,GAAG,2BAA2B,CAC9C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,QAAQ,EACR,KAAK,CAAC,YAAY,CACnB,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC;QAC1C,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAwB,EACxB,UAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,MAAM,QAAQ,GACZ,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACtE,IAAI,SAAS,GAAG,QAAQ,CAAC;IACzB,IAAI,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEhE,KAAK,IAAI,KAAK,GAAG,QAAQ,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAwB,EACxB,UAA8B,EAC9B,UAAkB;IAElB,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,SAAiB,EACjB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvC,MAAM,WAAW,GACf,SAAS,IAAI,QAAQ;YACnB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;QAEtD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI;YACJ,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,SAAS,GAAG,CAAC;gBACxB,OAAO,EAAE,SAAS,GAAG,CAAC;gBACtB,WAAW,EAAE,UAAU,GAAG,MAAM;gBAChC,SAAS,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM;aAC7C;SACF,CAAC,CAAC;QACH,MAAM,IAAI,WAAW,CAAC;IACxB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,WAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACtD,KAAK,EAAE;YACL,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,UAAU,GAAG,CAAC;YACzB,OAAO,EAAE,QAAQ,GAAG,CAAC;YACrB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC;YACpC,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM;SAC1D;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAwB;IACjD,MAAM,OAAO,GAAG,IAAI,KAAK,CAAU,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7D,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;QAEzC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACtB,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,2BAA2B,CAClC,KAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,YAAoB;IAEpB,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3D,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;YACzB,MAAM;QACR,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAChE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QAED,IAAI,KAAK,IAAI,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAgB;IACxC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,IAAI,IAAI;QACtC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI;QACrC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAC5E,CAAC;AACJ,CAAC"}
+{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../../src/engine/extraction/markdown/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAM9C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAgC,MAAM,cAAc,CAAC;AAChF,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAGjE,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAC1C,MAAM,oCAAoC,GAAG,GAAG,CAAC;AAoBjD,MAAM,OAAO,iBAAiB;IAC5B,KAAK,CAAC,OAAO,CACX,MAAc,EACd,UAAwB,EAAE;QAE1B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,YAAY,GAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE1D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,SAAS,GAAqB,EAAE,CAAC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,oBAAoB,CAAC;gBACnC,KAAK;gBACL,WAAW;gBACX,UAAU;gBACV,OAAO;gBACP,QAAQ,EAAE,YAAY,CAAC,aAAa;gBACpC,YAAY,EAAE,YAAY,CAAC,iBAAiB;aAC7C,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC1D,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE;oBACF,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,aAAa,CAClB,KAAK,EACL,WAAW,EACX,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,QAAQ,CACjB,CAAC,KAAK;oBACP,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,eAAe,CAAC,QAAQ,CAAC;qBAChC;oBACD,QAAQ;iBACT,CAAC,CAAC;gBAEH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,SAAS,CAAC,IAAI,CACZ,wBAAwB,CACtB,MAAM,EACN,QAAQ,EACR,MAAM,EACN,SAAS,CAAC,MAAM,EAChB,EAAE,CACH,CACF,CAAC;gBACJ,CAAC;gBAED,SAAS;YACX,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;oBAClD,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;oBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE;wBACP,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB;oBACD,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC1C,CAAC;IAEO,QAAQ,CACd,MAAkB,EAClB,OAA+B;QAE/B,OAAO,yBAAyB,CAC9B,MAAM,EACN,OAAO,CAAC,aAAa,EACrB,OAAO,CAAC,iBAAiB,CAC1B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,2BAA2B,CAClC,OAAqB;IAErB,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,4BAA4B,CAAC;IAC5E,MAAM,iBAAiB,GACrB,OAAO,CAAC,iBAAiB,IAAI,oCAAoC,CAAC;IAEpE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,WAAW,CACnB,2DAA2D,EAC3D;YACE,IAAI,EAAE,yDAAyD;YAC/D,OAAO,EAAE,iBAAiB,aAAa,EAAE;SAC1C,CACF,CAAC;IACJ,CAAC;IAED,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC;QACpC,iBAAiB,GAAG,CAAC;QACrB,iBAAiB,IAAI,aAAa,EAClC,CAAC;QACD,MAAM,IAAI,WAAW,CACnB,mEAAmE,EACnE;YACE,IAAI,EAAE,4DAA4D;YAClE,OAAO,EAAE,iBAAiB,aAAa,sBAAsB,iBAAiB,EAAE;SACjF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAAkB,EAClB,QAAgC,EAChC,MAAsB,EACtB,KAAa,EACb,KAAa;IAEb,OAAO;QACL,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;QACvC,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAgC;IACvD,OAAO,QAAQ,CAAC,OAAO,IAAI,kBAAkB,CAAC;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,KAAwB;IAC5C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACrD,IAAI,GAAG,EAAE,CAAC;YACR,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM;gBACpB,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gBACnB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;gBACjB,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B,EAC5B,KAAwB;IAExB,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEjC,IACE,YAAY,CAAC,SAAS,GAAG,CAAC;QAC1B,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EACnE,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,YAAY,CAAC,SAAS,GAAG,CAAC;YACpC,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;IACL,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEhC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAC1E,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,QAAQ,EACN,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACzB,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC;gBACnC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACtB,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3C,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,oBAAoB,CAAC,KAO7B;IACC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;IAE1C,OAAO,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YACxD,OAAO,CAAC,IAAI,CACV,GAAG,aAAa,CACd,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EACvB,UAAU,EACV,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,EAC7B,KAAK,CAAC,QAAQ,CACf,CACF,CAAC;YACF,UAAU,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,GAAG,UAAU,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACpD,IAAI,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBACrE,MAAM;YACR,CAAC;YACD,SAAS,IAAI,UAAU,CAAC;YACxB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,IAAI,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;YACpE,QAAQ,GAAG,mBAAmB,CAC5B,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,UAAU,EAChB,UAAU,EACV,QAAQ,CACT,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,IAAI,CACV,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CACxE,CAAC;QAEF,IAAI,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM;QACR,CAAC;QAED,MAAM,YAAY,GAAG,2BAA2B,CAC9C,KAAK,CAAC,KAAK,EACX,UAAU,EACV,QAAQ,EACR,KAAK,CAAC,YAAY,CACnB,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC;QAC1C,UAAU,GAAG,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAwB,EACxB,UAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,MAAM,QAAQ,GACZ,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACtE,IAAI,SAAS,GAAG,QAAQ,CAAC;IACzB,IAAI,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEhE,KAAK,IAAI,KAAK,GAAG,QAAQ,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAC3D,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAwB,EACxB,UAA8B,EAC9B,UAAkB;IAElB,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,SAAiB,EACjB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvC,MAAM,WAAW,GACf,SAAS,IAAI,QAAQ;YACnB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;QAEtD,OAAO,CAAC,IAAI,CAAC;YACX,IAAI;YACJ,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,SAAS,GAAG,CAAC;gBACxB,OAAO,EAAE,SAAS,GAAG,CAAC;gBACtB,WAAW,EAAE,UAAU,GAAG,MAAM;gBAChC,SAAS,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM;aAC7C;SACF,CAAC,CAAC;QACH,MAAM,IAAI,WAAW,CAAC;IACxB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,WAA8B,EAC9B,UAAkB,EAClB,QAAgB;IAEhB,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACtD,KAAK,EAAE;YACL,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,UAAU,GAAG,CAAC;YACzB,OAAO,EAAE,QAAQ,GAAG,CAAC;YACrB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC;YACpC,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM;SAC1D;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAwB;IACjD,MAAM,OAAO,GAAG,IAAI,KAAK,CAAU,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7D,IAAI,KAAK,GAAyB,IAAI,CAAC;IAEvC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;QAEzC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;YACtB,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,2BAA2B,CAClC,KAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,YAAoB;IAEpB,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3D,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;YACzB,MAAM;QACR,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAChE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QAED,IAAI,KAAK,IAAI,SAAS,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAgB;IACxC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,IAAI,IAAI;QACtC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI;QACrC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAC5E,CAAC;AACJ,CAAC"}

package/dist/engine/extraction/runtime.d.ts

@@ -0,0 +1,4 @@
+import type { EntityFragment } from "../types.js";
+import type { Source } from "./source.js";
+import type { ChunkOptions } from "./types.js";
+export declare function extract(source: Source, options?: ChunkOptions): Promise<EntityFragment[]>;

package/dist/engine/extraction/runtime.js

@@ -0,0 +1,26 @@
+import { CodeExtractor } from "./code/extractor.js";
+import { ImageExtractor } from "./image/extractor.js";
+import { MarkdownExtractor } from "./markdown/extractor.js";
+import { TextExtractor } from "./text/extractor.js";
+const extractors = {
+    code: new CodeExtractor(),
+    image: new ImageExtractor(),
+    markdown: new MarkdownExtractor(),
+    text: new TextExtractor(),
+};
+export function extract(source, options = {}) {
+    return extractors[routeSource(source)].extract(source, options);
+}
+function routeSource(source) {
+    if (source.kind === "image") {
+        return "image";
+    }
+    if (source.file.kind === "code") {
+        return "code";
+    }
+    if (source.file.format === "markdown") {
+        return "markdown";
+    }
+    return "text";
+}
+//# sourceMappingURL=runtime.js.map

package/dist/engine/extraction/runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../src/engine/extraction/runtime.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AASpD,MAAM,UAAU,GAAG;IACjB,IAAI,EAAE,IAAI,aAAa,EAAE;IACzB,KAAK,EAAE,IAAI,cAAc,EAAE;IAC3B,QAAQ,EAAE,IAAI,iBAAiB,EAAE;IACjC,IAAI,EAAE,IAAI,aAAa,EAAE;CACwB,CAAC;AAEpD,MAAM,UAAU,OAAO,CACrB,MAAc,EACd,UAAwB,EAAE;IAE1B,OAAO,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,MAAc;IACjC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACtC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/engine/extraction/source.d.ts

@@ -0,0 +1,18 @@
+import type { FileInfo, ImageFormat } from "../types.js";
+export type SourceKind = "text" | "image";
+type SourceBase = {
+    kind: SourceKind;
+    file: FileInfo;
+};
+export type TextSource = SourceBase & {
+    kind: "text";
+    text: string;
+};
+export type ImageSource = SourceBase & {
+    kind: "image";
+    data: Uint8Array;
+    format: ImageFormat;
+};
+export type Source = TextSource | ImageSource;
+export declare function validateSourceFile(source: Source): void;
+export {};

package/dist/engine/extraction/source.js

@@ -0,0 +1,22 @@
+import { EngineError } from "../errors.js";
+export function validateSourceFile(source) {
+    if (source.file.id.trim().length === 0) {
+        throw new EngineError("Extractor source requires a non-empty file id", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_FILE_ID",
+            context: `sourceKind=${source.kind}`,
+        });
+    }
+    if (source.file.absolutePath.trim().length === 0) {
+        throw new EngineError("Extractor source requires a non-empty absolute file path", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_ABSOLUTE_FILE_PATH",
+            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+        });
+    }
+    if (source.file.relativePath.trim().length === 0) {
+        throw new EngineError("Extractor source requires a non-empty relative file path", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.EMPTY_RELATIVE_FILE_PATH",
+            context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+        });
+    }
+}
+//# sourceMappingURL=source.js.map

package/dist/engine/extraction/source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.js","sourceRoot":"","sources":["../../../src/engine/extraction/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAuB3C,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;YACrE,IAAI,EAAE,2CAA2C;YACjD,OAAO,EAAE,cAAc,MAAM,CAAC,IAAI,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CACnB,0DAA0D,EAC1D;YACE,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CACnB,0DAA0D,EAC1D;YACE,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;SAC9D,CACF,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/engine/extraction/text/extractor.d.ts

@@ -0,0 +1,7 @@
+import type { EntityFragment } from "../../types.js";
+import { type Source, type TextSource } from "../source.js";
+import type { ChunkOptions } from "../types.js";
+export declare class TextExtractor {
+    extract(source: Source, options?: ChunkOptions): Promise<EntityFragment[]>;
+}
+export declare function extractPlainTextFragments(source: TextSource, maxChunkChars: number, chunkOverlapChars: number): EntityFragment[];

package/dist/engine/extraction/text/extractor.js

@@ -0,0 +1,168 @@
+import { EngineError } from "../../errors.js";
+import { makeEntityId } from "../ids.js";
+import { validateSourceFile } from "../source.js";
+const DEFAULT_TEXT_CHUNK_CHARS = 3600;
+const DEFAULT_TEXT_CHUNK_OVERLAP_CHARS = 540;
+export class TextExtractor {
+    async extract(source, options = {}) {
+        if (source.kind !== "text") {
+            throw new EngineError("Text extractor received a non-text source", {
+                code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_UNSUPPORTED_SOURCE",
+                context: `fileId=${source.file.id} sourceKind=${source.kind}`,
+            });
+        }
+        validateSourceFile(source);
+        const { maxChunkChars, chunkOverlapChars } = resolveTextChunkOptions(options);
+        return extractPlainTextFragments(source, maxChunkChars, chunkOverlapChars);
+    }
+}
+function resolveTextChunkOptions(options) {
+    const maxChunkChars = options.maxChunkChars ?? DEFAULT_TEXT_CHUNK_CHARS;
+    const chunkOverlapChars = options.chunkOverlapChars ?? DEFAULT_TEXT_CHUNK_OVERLAP_CHARS;
+    if (!Number.isInteger(maxChunkChars) || maxChunkChars <= 0) {
+        throw new EngineError("Text extractor requires a positive integer chunk size", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_INVALID_CHUNK_SIZE",
+            context: `maxChunkChars=${maxChunkChars}`,
+        });
+    }
+    if (!Number.isInteger(chunkOverlapChars) ||
+        chunkOverlapChars < 0 ||
+        chunkOverlapChars >= maxChunkChars) {
+        throw new EngineError("Text extractor requires overlap to be smaller than chunk size", {
+            code: "ZVEC_GREP.ENGINE.EXTRACTORS.TEXT_INVALID_CHUNK_OVERLAP",
+            context: `maxChunkChars=${maxChunkChars} chunkOverlapChars=${chunkOverlapChars}`,
+        });
+    }
+    return { maxChunkChars, chunkOverlapChars };
+}
+export function extractPlainTextFragments(source, maxChunkChars, chunkOverlapChars) {
+    return chunkText(source.text, maxChunkChars, chunkOverlapChars).map((chunk, index) => ({
+        id: makeEntityId(source.file.id, index),
+        fileId: source.file.id,
+        range: chunk.range,
+        content: {
+            kind: "text",
+            text: chunk.text,
+        },
+    }));
+}
+function chunkText(text, maxChars, overlapChars) {
+    if (text.trim().length === 0) {
+        return [];
+    }
+    const lines = text.split("\n");
+    const lineOffsets = computeLineOffsets(lines);
+    const chunks = [];
+    let startIndex = 0;
+    while (startIndex < lines.length) {
+        if (lines[startIndex].length + 1 > maxChars) {
+            const line = lines[startIndex];
+            const lineNumber = startIndex + 1;
+            let offset = 0;
+            while (offset < line.length) {
+                const remaining = line.length - offset;
+                const sliceLength = remaining <= maxChars
+                    ? remaining
+                    : findLineCut(line.slice(offset), maxChars);
+                const slice = line.slice(offset, offset + sliceLength);
+                if (slice.trim().length > 0) {
+                    chunks.push({
+                        text: slice,
+                        range: {
+                            kind: "text",
+                            startLine: lineNumber,
+                            endLine: lineNumber,
+                            startOffset: lineOffsets[startIndex] + offset,
+                            endOffset: lineOffsets[startIndex] + offset + slice.length,
+                        },
+                    });
+                }
+                offset += sliceLength;
+            }
+            startIndex++;
+            continue;
+        }
+        let usedChars = 0;
+        let endIndex = startIndex;
+        while (endIndex < lines.length) {
+            const lineLength = lines[endIndex].length + 1;
+            if (usedChars + lineLength > maxChars && endIndex > startIndex) {
+                break;
+            }
+            usedChars += lineLength;
+            endIndex++;
+        }
+        const chunkLines = lines.slice(startIndex, endIndex);
+        const chunk = chunkLines.join("\n");
+        if (chunk.trim().length > 0) {
+            const endLineIndex = endIndex - 1;
+            chunks.push({
+                text: chunk,
+                range: {
+                    kind: "text",
+                    startLine: startIndex + 1,
+                    endLine: endIndex,
+                    startOffset: lineOffsets[startIndex],
+                    endOffset: lineOffsets[endLineIndex] + lines[endLineIndex].length,
+                },
+            });
+        }
+        if (endIndex >= lines.length) {
+            break;
+        }
+        startIndex = computeNextStartLine(lines, startIndex, endIndex, overlapChars);
+    }
+    return chunks;
+}
+function computeLineOffsets(lines) {
+    const offsets = [];
+    let offset = 0;
+    for (const line of lines) {
+        offsets.push(offset);
+        offset += line.length + 1;
+    }
+    return offsets;
+}
+function computeNextStartLine(lines, startIndex, endIndex, overlapChars) {
+    if (overlapChars === 0) {
+        return endIndex;
+    }
+    let overlapLines = 0;
+    let overlapCount = 0;
+    for (let index = endIndex - 1; index >= startIndex && overlapCount < overlapChars; index--) {
+        overlapCount += lines[index].length + 1;
+        overlapLines++;
+    }
+    const nextStart = endIndex - overlapLines;
+    return nextStart > startIndex ? nextStart : endIndex;
+}
+function findLineCut(line, maxChars) {
+    if (line.length <= maxChars) {
+        return line.length;
+    }
+    const minPosition = Math.floor(maxChars * 0.7);
+    let bestPosition = -1;
+    let bestScore = 0;
+    for (let index = minPosition; index < maxChars; index++) {
+        const character = line[index];
+        let score = 0;
+        if (character === "." || character === "!" || character === "?") {
+            score = 4;
+        }
+        else if (character === "," || character === ";" || character === ":") {
+            score = 3;
+        }
+        else if (character === " " || character === "\t") {
+            score = 2;
+        }
+        else if (character === "-" || character === "/" || character === "\\") {
+            score = 1;
+        }
+        if (score > 0 && score >= bestScore) {
+            bestScore = score;
+            bestPosition = index + 1;
+        }
+    }
+    return bestPosition > 0 ? bestPosition : maxChars;
+}
+//# sourceMappingURL=extractor.js.map

package/dist/engine/extraction/text/extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../../../src/engine/extraction/text/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAgC,MAAM,cAAc,CAAC;AAGhF,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAEtC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAE7C,MAAM,OAAO,aAAa;IACxB,KAAK,CAAC,OAAO,CACX,MAAc,EACd,UAAwB,EAAE;QAE1B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,WAAW,CAAC,2CAA2C,EAAE;gBACjE,IAAI,EAAE,qDAAqD;gBAC3D,OAAO,EAAE,UAAU,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE;aAC9D,CAAC,CAAC;QACL,CAAC;QAED,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,GACxC,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO,yBAAyB,CAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAC7E,CAAC;CACF;AAED,SAAS,uBAAuB,CAC9B,OAAqB;IAErB,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,wBAAwB,CAAC;IACxE,MAAM,iBAAiB,GACrB,OAAO,CAAC,iBAAiB,IAAI,gCAAgC,CAAC;IAEhE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,WAAW,CACnB,uDAAuD,EACvD;YACE,IAAI,EAAE,qDAAqD;YAC3D,OAAO,EAAE,iBAAiB,aAAa,EAAE;SAC1C,CACF,CAAC;IACJ,CAAC;IAED,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC;QACpC,iBAAiB,GAAG,CAAC;QACrB,iBAAiB,IAAI,aAAa,EAClC,CAAC;QACD,MAAM,IAAI,WAAW,CACnB,+DAA+D,EAC/D;YACE,IAAI,EAAE,wDAAwD;YAC9D,OAAO,EAAE,iBAAiB,aAAa,sBAAsB,iBAAiB,EAAE;SACjF,CACF,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,MAAkB,EAClB,aAAqB,EACrB,iBAAyB;IAEzB,OAAO,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,iBAAiB,CAAC,CAAC,GAAG,CACjE,CAAC,KAAK,EAAE,KAAK,EAAkB,EAAE,CAAC,CAAC;QACjC,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC;QACvC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QACtB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB;KACF,CAAC,CACH,CAAC;AACJ,CAAC;AAOD,SAAS,SAAS,CAChB,IAAY,EACZ,QAAgB,EAChB,YAAoB;IAEpB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,EAAE,CAAC;YAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;YAC/B,MAAM,UAAU,GAAG,UAAU,GAAG,CAAC,CAAC;YAClC,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvC,MAAM,WAAW,GACf,SAAS,IAAI,QAAQ;oBACnB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;gBAEvD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE;4BACL,IAAI,EAAE,MAAM;4BACZ,SAAS,EAAE,UAAU;4BACrB,OAAO,EAAE,UAAU;4BACnB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM;4BAC7C,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,KAAK,CAAC,MAAM;yBAC3D;qBACF,CAAC,CAAC;gBACL,CAAC;gBAED,MAAM,IAAI,WAAW,CAAC;YACxB,CAAC;YAED,UAAU,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,QAAQ,GAAG,UAAU,CAAC;QAE1B,OAAO,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAE9C,IAAI,SAAS,GAAG,UAAU,GAAG,QAAQ,IAAI,QAAQ,GAAG,UAAU,EAAE,CAAC;gBAC/D,MAAM;YACR,CAAC;YAED,SAAS,IAAI,UAAU,CAAC;YACxB,QAAQ,EAAE,CAAC;QACb,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,QAAQ,GAAG,CAAC,CAAC;YAElC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,UAAU,GAAG,CAAC;oBACzB,OAAO,EAAE,QAAQ;oBACjB,WAAW,EAAE,WAAW,CAAC,UAAU,CAAC;oBACpC,SAAS,EAAE,WAAW,CAAC,YAAY,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,MAAM;iBAClE;aACF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM;QACR,CAAC;QAED,UAAU,GAAG,oBAAoB,CAC/B,KAAK,EACL,UAAU,EACV,QAAQ,EACR,YAAY,CACb,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,YAAoB;IAEpB,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KACE,IAAI,KAAK,GAAG,QAAQ,GAAG,CAAC,EACxB,KAAK,IAAI,UAAU,IAAI,YAAY,GAAG,YAAY,EAClD,KAAK,EAAE,EACP,CAAC;QACD,YAAY,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC;IAE1C,OAAO,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvD,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YAChE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,GAAG,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACxE,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC"}

package/dist/engine/extraction/types.d.ts

@@ -0,0 +1,4 @@
+export type ChunkOptions = {
+    maxChunkChars?: number;
+    chunkOverlapChars?: number;
+};

package/dist/engine/{models → extraction}/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/engine/models/types.ts"],"names":[],"mappings":""}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/engine/extraction/types.ts"],"names":[],"mappings":""}

package/dist/engine/{files/file-type.d.ts → file-type.d.ts}

@@ -1,4 +1,4 @@
-import type { FileFormat, FileKind } from "../types.js";
+import type { FileFormat, FileKind } from "./types.js";
 export declare function detectFileType(path: string): {
     kind: FileKind;
     format: FileFormat;

package/dist/engine/file-type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-type.js","sourceRoot":"","sources":["../../src/engine/file-type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG9C,MAAM,YAAY,GAA2B;IAC3C,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,KAAK;IACX,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IACb,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,QAAQ;IACf,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,MAAM;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,SAAS,EAAE,QAAQ;CACpB,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,MAAM;IAChB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;CACf,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,KAAK,EAAE,UAAU;IACjB,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;CACd,CAAC;AAEF,MAAM,aAAa,GAAgC;IACjD,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,QAAQ;IACR,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,UAAU,cAAc,CAC5B,IAAY;IAEZ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAChD,CAAC;IAED,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAE9C,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,aAAa,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC"}

package/dist/engine/index-status.d.ts

@@ -0,0 +1,9 @@
+import type { WorkspaceIndexStatus, IndexProgress } from "./types.js";
+export type IndexCompletion = {
+    completed: number;
+    total: number;
+};
+export declare function indexStatusNeedsRefresh(status: WorkspaceIndexStatus | null | undefined): boolean;
+export declare function indexCompletionFromStatus(status: WorkspaceIndexStatus | null | undefined): IndexCompletion | undefined;
+export declare function mergeIndexCompletion(completion: IndexCompletion | undefined, progress: IndexProgress | undefined): IndexCompletion | undefined;
+export declare function indexCompletionForJob(completion: IndexCompletion | undefined, state: "queued" | "running" | "succeeded" | "failed" | "cancelled" | undefined, progress: IndexProgress | undefined): IndexCompletion | undefined;

package/dist/engine/index-status.js

@@ -0,0 +1,38 @@
+export function indexStatusNeedsRefresh(status) {
+    return Boolean(status &&
+        (status.filesAdded > 0 ||
+            status.filesModified > 0 ||
+            status.filesDeleted > 0 ||
+            status.filesPending > 0 ||
+            status.filesFailed > 0));
+}
+export function indexCompletionFromStatus(status) {
+    if (!status)
+        return undefined;
+    return {
+        completed: status.filesUnchanged,
+        total: status.filesScanned,
+    };
+}
+export function mergeIndexCompletion(completion, progress) {
+    const processed = progress?.filesIndexed;
+    const total = progress?.filesTotal;
+    const succeeded = processed === undefined
+        ? undefined
+        : Math.max(0, processed - (progress?.filesFailed ?? 0));
+    if (!completion) {
+        return succeeded === undefined || total === undefined
+            ? undefined
+            : { completed: succeeded, total };
+    }
+    return {
+        completed: Math.min(completion.total, completion.completed + (succeeded ?? 0)),
+        total: completion.total,
+    };
+}
+export function indexCompletionForJob(completion, state, progress) {
+    return state === "running"
+        ? mergeIndexCompletion(completion, progress)
+        : completion;
+}
+//# sourceMappingURL=index-status.js.map

package/dist/engine/index-status.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-status.js","sourceRoot":"","sources":["../../src/engine/index-status.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,uBAAuB,CACrC,MAA+C;IAE/C,OAAO,OAAO,CACZ,MAAM;QACN,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC;YACpB,MAAM,CAAC,aAAa,GAAG,CAAC;YACxB,MAAM,CAAC,YAAY,GAAG,CAAC;YACvB,MAAM,CAAC,YAAY,GAAG,CAAC;YACvB,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,MAA+C;IAE/C,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,cAAc;QAChC,KAAK,EAAE,MAAM,CAAC,YAAY;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,UAAuC,EACvC,QAAmC;IAEnC,MAAM,SAAS,GAAG,QAAQ,EAAE,YAAY,CAAC;IACzC,MAAM,KAAK,GAAG,QAAQ,EAAE,UAAU,CAAC;IACnC,MAAM,SAAS,GACb,SAAS,KAAK,SAAS;QACrB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,SAAS,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS;YACnD,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACtC,CAAC;IACD,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,CACjB,UAAU,CAAC,KAAK,EAChB,UAAU,CAAC,SAAS,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC,CACxC;QACD,KAAK,EAAE,UAAU,CAAC,KAAK;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,UAAuC,EACvC,KACyE,EACzE,QAAmC;IAEnC,OAAO,KAAK,KAAK,SAAS;QACxB,CAAC,CAAC,oBAAoB,CAAC,UAAU,EAAE,QAAQ,CAAC;QAC5C,CAAC,CAAC,UAAU,CAAC;AACjB,CAAC"}

package/dist/engine/manifest.d.ts

@@ -0,0 +1,12 @@
+import type { EmbeddingRuntimeConfig } from "./config.js";
+import type { WorkspaceIndexInfo } from "./types.js";
+export declare const WORKSPACE_MANIFEST_FILE = "manifest.json";
+export declare const CURRENT_MANIFEST_VERSION = 1;
+export type WorkspaceManifest = WorkspaceIndexInfo & {
+    manifestVersion: typeof CURRENT_MANIFEST_VERSION;
+    embeddingRuntime: EmbeddingRuntimeConfig;
+};
+export declare function workspaceManifestPath(home: string): string;
+export declare function readWorkspaceManifest(home: string): WorkspaceManifest | null;
+export declare function writeWorkspaceManifest(home: string, manifest: WorkspaceManifest): void;
+export declare function workspaceIndexInfoFromManifest(manifest: WorkspaceManifest): WorkspaceIndexInfo;

package/dist/engine/manifest.js

@@ -0,0 +1,120 @@
+import { join } from "node:path";
+import { EngineError } from "./errors.js";
+import { readJsonFileSync, writeJsonFileSync } from "./utils/json.js";
+export const WORKSPACE_MANIFEST_FILE = "manifest.json";
+export const CURRENT_MANIFEST_VERSION = 1;
+const WORKSPACE_DIRECTORY_MODE = 0o700;
+const WORKSPACE_MANIFEST_MODE = 0o600;
+export function workspaceManifestPath(home) {
+    return join(home, WORKSPACE_MANIFEST_FILE);
+}
+export function readWorkspaceManifest(home) {
+    const path = workspaceManifestPath(home);
+    const value = readJsonFileSync(path, null);
+    if (value === null) {
+        return null;
+    }
+    if (!isWorkspaceManifest(value)) {
+        throw new EngineError("Workspace index manifest is invalid", {
+            code: "ZVEC_GREP.ENGINE.MANIFEST.INVALID",
+            context: `path=${path}`,
+        });
+    }
+    return value;
+}
+export function writeWorkspaceManifest(home, manifest) {
+    writeJsonFileSync(workspaceManifestPath(home), manifest, {
+        directoryMode: WORKSPACE_DIRECTORY_MODE,
+        fileMode: WORKSPACE_MANIFEST_MODE,
+    });
+}
+export function workspaceIndexInfoFromManifest(manifest) {
+    return {
+        id: manifest.id,
+        name: manifest.name,
+        path: manifest.path,
+        rootPaths: manifest.rootPaths,
+        indexPolicy: manifest.indexPolicy,
+        embedding: manifest.embedding,
+        indexVersion: manifest.indexVersion,
+        createdTime: manifest.createdTime,
+        updatedTime: manifest.updatedTime,
+    };
+}
+function isWorkspaceManifest(value) {
+    if (!isRecord(value) || value.manifestVersion !== CURRENT_MANIFEST_VERSION) {
+        return false;
+    }
+    return (isNonEmptyString(value.id) &&
+        isNonEmptyString(value.name) &&
+        isNonEmptyString(value.path) &&
+        Array.isArray(value.rootPaths) &&
+        value.rootPaths.length > 0 &&
+        value.rootPaths.every(isRootPath) &&
+        (value.indexPolicy === "enabled" || value.indexPolicy === "disabled") &&
+        (value.embedding === null || isEmbeddingSchema(value.embedding)) &&
+        (value.indexVersion === null || Number.isInteger(value.indexVersion)) &&
+        typeof value.createdTime === "number" &&
+        Number.isFinite(value.createdTime) &&
+        typeof value.updatedTime === "number" &&
+        Number.isFinite(value.updatedTime) &&
+        isEmbeddingRuntime(value.embeddingRuntime));
+}
+function isRootPath(value) {
+    return (isRecord(value) &&
+        isNonEmptyString(value.absolutePath) &&
+        typeof value.recursive === "boolean" &&
+        isOptionalStringArray(value.include) &&
+        isOptionalStringArray(value.exclude) &&
+        isOptionalStringArray(value.globs) &&
+        isOptionalStringArray(value.insensitiveGlobs) &&
+        isOptionalStringArray(value.fileTypes) &&
+        isOptionalStringArray(value.excludedFileTypes) &&
+        isOptionalBoolean(value.hidden) &&
+        isOptionalBoolean(value.noIgnore) &&
+        isOptionalStringArray(value.ignoreFiles) &&
+        isOptionalNonNegativeInteger(value.maxDepth) &&
+        isOptionalNonNegativeInteger(value.maxFileSizeBytes) &&
+        isOptionalBoolean(value.follow));
+}
+function isEmbeddingSchema(value) {
+    return (isRecord(value) &&
+        isNonEmptyString(value.provider) &&
+        isNonEmptyString(value.model) &&
+        Number.isInteger(value.dimension) &&
+        typeof value.dimension === "number" &&
+        value.dimension > 0 &&
+        (value.metric === "cosine" ||
+            value.metric === "dot" ||
+            value.metric === "euclidean"));
+}
+function isEmbeddingRuntime(value) {
+    if (!isRecord(value)) {
+        return false;
+    }
+    return ((value.apiKey === undefined || typeof value.apiKey === "string") &&
+        (value.endpoint === undefined || typeof value.endpoint === "string") &&
+        (value.device === undefined ||
+            value.device === "auto" ||
+            value.device === "cpu" ||
+            value.device === "metal" ||
+            value.device === "vulkan" ||
+            value.device === "cuda"));
+}
+function isOptionalStringArray(value) {
+    return (value === undefined ||
+        (Array.isArray(value) && value.every((item) => typeof item === "string")));
+}
+function isOptionalBoolean(value) {
+    return value === undefined || typeof value === "boolean";
+}
+function isOptionalNonNegativeInteger(value) {
+    return value === undefined || (Number.isInteger(value) && Number(value) >= 0);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isNonEmptyString(value) {
+    return typeof value === "string" && value.length > 0;
+}
+//# sourceMappingURL=manifest.js.map

package/dist/engine/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/engine/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEtE,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC;AACvD,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAE1C,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAOtC,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,OAAO,IAAI,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,gBAAgB,CAAU,IAAI,EAAE,IAAI,CAAC,CAAC;IACpD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,WAAW,CAAC,qCAAqC,EAAE;YAC3D,IAAI,EAAE,mCAAmC;YACzC,OAAO,EAAE,QAAQ,IAAI,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAY,EACZ,QAA2B;IAE3B,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE;QACvD,aAAa,EAAE,wBAAwB;QACvC,QAAQ,EAAE,uBAAuB;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,QAA2B;IAE3B,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,eAAe,KAAK,wBAAwB,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC;QAC9B,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;QAC1B,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC;QACjC,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,KAAK,UAAU,CAAC;QACrE,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChE,CAAC,KAAK,CAAC,YAAY,KAAK,IAAI,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrE,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;QAClC,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;QAClC,kBAAkB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,gBAAgB,CAAC,KAAK,CAAC,YAAY,CAAC;QACpC,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS;QACpC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC;QAClC,qBAAqB,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAC7C,qBAAqB,CAAC,KAAK,CAAC,SAAS,CAAC;QACtC,qBAAqB,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC9C,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/B,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjC,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC;QACxC,4BAA4B,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,4BAA4B,CAAC,KAAK,CAAC,gBAAgB,CAAC;QACpD,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAChC,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC;QACjC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,KAAK,CAAC,SAAS,GAAG,CAAC;QACnB,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ;YACxB,KAAK,CAAC,MAAM,KAAK,KAAK;YACtB,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC;QAChE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC;QACpE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;YACzB,KAAK,CAAC,MAAM,KAAK,MAAM;YACvB,KAAK,CAAC,MAAM,KAAK,KAAK;YACtB,KAAK,CAAC,MAAM,KAAK,OAAO;YACxB,KAAK,CAAC,MAAM,KAAK,QAAQ;YACzB,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAC3B,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,OAAO,CACL,KAAK,KAAK,SAAS;QACnB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAC1E,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,SAAS,CAAC;AAC3D,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAc;IAClD,OAAO,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACvD,CAAC"}