expxagents 0.25.2 → 0.25.4

package/node_modules/expxagents-knowledge/dist/ingest/chunker.js

@@ -0,0 +1,221 @@
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export function chunkMarkdown(content, options) {
+    if (!content.trim())
+        return [];
+    const sections = splitByHeadings(content);
+    const results = [];
+    let chunkIndex = 0;
+    for (const section of sections) {
+        const sectionTokens = estimateTokens(section.content);
+        if (sectionTokens <= options.max_tokens) {
+            results.push({
+                content: section.content,
+                chunk_index: chunkIndex++,
+                token_count: sectionTokens,
+                metadata: { heading_path: section.heading_path },
+            });
+        }
+        else {
+            const subChunks = splitByParagraphs(section.content, options);
+            for (const sub of subChunks) {
+                results.push({
+                    content: sub,
+                    chunk_index: chunkIndex++,
+                    token_count: estimateTokens(sub),
+                    metadata: { heading_path: section.heading_path },
+                });
+            }
+        }
+    }
+    return results;
+}
+function splitByHeadings(content) {
+    const lines = content.split('\n');
+    const sections = [];
+    let currentHeadings = [];
+    let currentLines = [];
+    for (const line of lines) {
+        const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+        if (headingMatch) {
+            if (currentLines.length > 0) {
+                const text = currentLines.join('\n').trim();
+                if (text) {
+                    sections.push({
+                        heading_path: currentHeadings.join(' > ') || '',
+                        content: text,
+                    });
+                }
+                currentLines = [];
+            }
+            const level = headingMatch[1].length;
+            const title = headingMatch[2].trim();
+            currentHeadings = currentHeadings.slice(0, level - 1);
+            currentHeadings[level - 1] = title;
+            currentHeadings = currentHeadings.slice(0, level);
+            currentLines.push(line);
+        }
+        else {
+            currentLines.push(line);
+        }
+    }
+    if (currentLines.length > 0) {
+        const text = currentLines.join('\n').trim();
+        if (text) {
+            sections.push({
+                heading_path: currentHeadings.join(' > ') || '',
+                content: text,
+            });
+        }
+    }
+    return sections;
+}
+function splitByParagraphs(content, options) {
+    const blocks = splitPreservingCodeBlocks(content);
+    const chunks = [];
+    let current = [];
+    let currentTokens = 0;
+    for (const block of blocks) {
+        const blockTokens = estimateTokens(block);
+        if (currentTokens + blockTokens > options.max_tokens && current.length > 0) {
+            chunks.push(current.join('\n\n'));
+            if (options.overlap > 0) {
+                const overlapTokens = Math.floor(options.max_tokens * options.overlap);
+                const kept = [];
+                let keptTokens = 0;
+                for (let i = current.length - 1; i >= 0; i--) {
+                    const t = estimateTokens(current[i]);
+                    if (keptTokens + t > overlapTokens)
+                        break;
+                    kept.unshift(current[i]);
+                    keptTokens += t;
+                }
+                current = kept;
+                currentTokens = keptTokens;
+            }
+            else {
+                current = [];
+                currentTokens = 0;
+            }
+        }
+        current.push(block);
+        currentTokens += blockTokens;
+    }
+    if (current.length > 0) {
+        chunks.push(current.join('\n\n'));
+    }
+    return chunks;
+}
+const PAGE_SEPARATOR_REGEX = /^\[PAGE (\d+)\]$/;
+export function chunkPlainText(content, options) {
+    if (!content.trim())
+        return [];
+    const pageSections = splitByPages(content);
+    const results = [];
+    let chunkIndex = 0;
+    for (const section of pageSections) {
+        const blocks = section.content.split(/\n\n/).filter(b => b.trim());
+        const chunks = [];
+        let current = [];
+        let currentTokens = 0;
+        for (const block of blocks) {
+            const blockTokens = estimateTokens(block);
+            if (currentTokens + blockTokens > options.max_tokens && current.length > 0) {
+                chunks.push(current.join('\n\n'));
+                if (options.overlap > 0) {
+                    const overlapTokens = Math.floor(options.max_tokens * options.overlap);
+                    const kept = [];
+                    let keptTokens = 0;
+                    for (let i = current.length - 1; i >= 0; i--) {
+                        const t = estimateTokens(current[i]);
+                        if (keptTokens + t > overlapTokens)
+                            break;
+                        kept.unshift(current[i]);
+                        keptTokens += t;
+                    }
+                    current = kept;
+                    currentTokens = keptTokens;
+                }
+                else {
+                    current = [];
+                    currentTokens = 0;
+                }
+            }
+            current.push(block);
+            currentTokens += blockTokens;
+        }
+        if (current.length > 0) {
+            chunks.push(current.join('\n\n'));
+        }
+        for (const chunk of chunks) {
+            results.push({
+                content: chunk,
+                chunk_index: chunkIndex++,
+                token_count: estimateTokens(chunk),
+                metadata: { page_number: section.page },
+            });
+        }
+    }
+    return results;
+}
+function splitByPages(content) {
+    const lines = content.split('\n');
+    const sections = [];
+    let currentLines = [];
+    let currentPage = 1;
+    for (const line of lines) {
+        const match = line.match(PAGE_SEPARATOR_REGEX);
+        if (match) {
+            if (currentLines.length > 0) {
+                const text = currentLines.join('\n').trim();
+                if (text) {
+                    sections.push({ page: currentPage, content: text });
+                }
+            }
+            // [PAGE N] marks the start of page N content
+            currentPage = parseInt(match[1], 10);
+            currentLines = [];
+        }
+        else {
+            currentLines.push(line);
+        }
+    }
+    if (currentLines.length > 0) {
+        const text = currentLines.join('\n').trim();
+        if (text) {
+            sections.push({ page: currentPage, content: text });
+        }
+    }
+    return sections.length > 0 ? sections : [{ page: 1, content }];
+}
+function splitPreservingCodeBlocks(content) {
+    const blocks = [];
+    const parts = content.split(/\n\n/);
+    let inCodeBlock = false;
+    let codeBuffer = [];
+    for (const part of parts) {
+        const fenceCount = (part.match(/```/g) || []).length;
+        if (inCodeBlock) {
+            codeBuffer.push(part);
+            if (fenceCount % 2 === 1) {
+                blocks.push(codeBuffer.join('\n\n'));
+                codeBuffer = [];
+                inCodeBlock = false;
+            }
+        }
+        else if (fenceCount % 2 === 1) {
+            inCodeBlock = true;
+            codeBuffer.push(part);
+        }
+        else {
+            if (part.trim())
+                blocks.push(part);
+        }
+    }
+    if (codeBuffer.length > 0) {
+        blocks.push(codeBuffer.join('\n\n'));
+    }
+    return blocks;
+}
+//# sourceMappingURL=chunker.js.map

package/node_modules/expxagents-knowledge/dist/ingest/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/ingest/chunker.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAOD,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,OAAuB;IACpE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAE/B,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEtD,IAAI,aAAa,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,WAAW,EAAE,UAAU,EAAE;gBACzB,WAAW,EAAE,aAAa;gBAC1B,QAAQ,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE;aACjD,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9D,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,GAAG;oBACZ,WAAW,EAAE,UAAU,EAAE;oBACzB,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC;oBAChC,QAAQ,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,OAAe;IACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,YAAY,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,YAAY,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;wBAC/C,OAAO,EAAE,IAAI;qBACd,CAAC,CAAC;gBACL,CAAC;gBACD,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACrC,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACrC,eAAe,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACtD,eAAe,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;YACnC,eAAe,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAClD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC;gBACZ,YAAY,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;gBAC/C,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe,EAAE,OAAuB;IACjE,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAE1C,IAAI,aAAa,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAClC,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;gBACnB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,UAAU,GAAG,CAAC,GAAG,aAAa;wBAAE,MAAM;oBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzB,UAAU,IAAI,CAAC,CAAC;gBAClB,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,aAAa,GAAG,UAAU,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,EAAE,CAAC;gBACb,aAAa,GAAG,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,aAAa,IAAI,WAAW,CAAC;IAC/B,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AAEhD,MAAM,UAAU,cAAc,CAAC,OAAe,EAAE,OAAuB;IACrE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAE/B,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAa,EAAE,CAAC;QAC3B,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAE1C,IAAI,aAAa,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBAElC,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;oBACvE,MAAM,IAAI,GAAa,EAAE,CAAC;oBAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;oBACnB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;wBACrC,IAAI,UAAU,GAAG,CAAC,GAAG,aAAa;4BAAE,MAAM;wBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;wBACzB,UAAU,IAAI,CAAC,CAAC;oBAClB,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC;oBACf,aAAa,GAAG,UAAU,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,EAAE,CAAC;oBACb,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,aAAa,IAAI,WAAW,CAAC;QAC/B,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACpC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,UAAU,EAAE;gBACzB,WAAW,EAAE,cAAc,CAAC,KAAK,CAAC;gBAClC,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,IAAI,EAAE;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC/C,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;YACD,6CAA6C;YAC7C,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,YAAY,GAAG,EAAE,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,yBAAyB,CAAC,OAAe;IAChD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAErD,IAAI,WAAW,EAAE,CAAC;YAChB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBACrC,UAAU,GAAG,EAAE,CAAC;gBAChB,WAAW,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,WAAW,GAAG,IAAI,CAAC;YACnB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,IAAI,EAAE;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/node_modules/expxagents-knowledge/dist/ingest/document-loader.d.ts

@@ -0,0 +1,4 @@
+import type { ParsedDocument, PdfConfig } from '../types.js';
+export declare function computeHash(content: string | Buffer): string;
+export declare function loadDocument(filePath: string, maxSizeMb?: number, pdfConfig?: PdfConfig): Promise<ParsedDocument>;
+//# sourceMappingURL=document-loader.d.ts.map

package/node_modules/expxagents-knowledge/dist/ingest/document-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-loader.d.ts","sourceRoot":"","sources":["../../src/ingest/document-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAsB7D,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAE5D;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,EAChB,SAAS,SAAK,EACd,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,cAAc,CAAC,CA2BzB"}

package/node_modules/expxagents-knowledge/dist/ingest/document-loader.js

@@ -0,0 +1,56 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { extractPdf } from './pdf-extractor.js';
+const MIME_MAP = {
+    '.md': 'text/markdown',
+    '.markdown': 'text/markdown',
+    '.txt': 'text/plain',
+    '.json': 'application/json',
+    '.yaml': 'text/yaml',
+    '.yml': 'text/yaml',
+    '.pdf': 'application/pdf',
+};
+const SUPPORTED_EXTENSIONS = new Set(Object.keys(MIME_MAP));
+const DEFAULT_PDF_CONFIG = {
+    ocr_enabled: true,
+    ocr_languages: ['eng'],
+    ocr_threshold: 50,
+    max_pages: 500,
+};
+export function computeHash(content) {
+    return crypto.createHash('sha256').update(content).digest('hex');
+}
+export async function loadDocument(filePath, maxSizeMb = 10, pdfConfig) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (!SUPPORTED_EXTENSIONS.has(ext)) {
+        throw new Error(`Unsupported file type: ${ext}`);
+    }
+    const stat = fs.statSync(filePath);
+    const sizeMb = stat.size / (1024 * 1024);
+    if (sizeMb > maxSizeMb) {
+        throw new Error(`File too large: ${sizeMb.toFixed(1)}MB exceeds limit of ${maxSizeMb}MB`);
+    }
+    if (ext === '.pdf') {
+        const result = await extractPdf(filePath, pdfConfig ?? DEFAULT_PDF_CONFIG);
+        return {
+            title: path.basename(filePath, '.pdf'),
+            content: result.content,
+            mime_type: 'application/pdf',
+            metadata: result.metadata,
+        };
+    }
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const mime_type = MIME_MAP[ext];
+    const title = extractTitle(content, filePath, mime_type);
+    return { title, content, mime_type };
+}
+function extractTitle(content, filePath, mimeType) {
+    if (mimeType === 'text/markdown') {
+        const match = content.match(/^#{1,6}\s+(.+)$/m);
+        if (match)
+            return match[1].trim();
+    }
+    return path.basename(filePath);
+}
+//# sourceMappingURL=document-loader.js.map

package/node_modules/expxagents-knowledge/dist/ingest/document-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-loader.js","sourceRoot":"","sources":["../../src/ingest/document-loader.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,QAAQ,GAA2B;IACvC,KAAK,EAAE,eAAe;IACtB,WAAW,EAAE,eAAe;IAC5B,MAAM,EAAE,YAAY;IACpB,OAAO,EAAE,kBAAkB;IAC3B,OAAO,EAAE,WAAW;IACpB,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,iBAAiB;CAC1B,CAAC;AAEF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE5D,MAAM,kBAAkB,GAAc;IACpC,WAAW,EAAE,IAAI;IACjB,aAAa,EAAE,CAAC,KAAK,CAAC;IACtB,aAAa,EAAE,EAAE;IACjB,SAAS,EAAE,GAAG;CACf,CAAC;AAEF,MAAM,UAAU,WAAW,CAAC,OAAwB;IAClD,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAgB,EAChB,SAAS,GAAG,EAAE,EACd,SAAqB;IAErB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACzC,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,SAAS,IAAI,CAAC,CAAC;IAC5F,CAAC;IAED,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,SAAS,IAAI,kBAAkB,CAAC,CAAC;QAC3E,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;YACtC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,iBAAiB;YAC5B,QAAQ,EAAE,MAAM,CAAC,QAA8C;SAChE,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAE,CAAC;IACjC,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAEzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,OAAe,EAAE,QAAgB,EAAE,QAAgB;IACvE,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAChD,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACjC,CAAC"}

package/node_modules/expxagents-knowledge/dist/ingest/embedder.d.ts

@@ -0,0 +1,4 @@
+import type { Embedder } from '../types.js';
+export declare function createMockEmbedder(dims: number): Embedder;
+export declare function createEmbedder(provider: string): Embedder;
+//# sourceMappingURL=embedder.d.ts.map

package/node_modules/expxagents-knowledge/dist/ingest/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/ingest/embedder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAiBzD;AAGD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAGzD"}

package/node_modules/expxagents-knowledge/dist/ingest/embedder.js

@@ -0,0 +1,25 @@
+import crypto from 'node:crypto';
+export function createMockEmbedder(dims) {
+    return {
+        dimensions: dims,
+        async embed(texts) {
+            return texts.map((text) => {
+                // Deterministic hash-based vector for testing
+                const hash = crypto.createHash('sha256').update(text).digest();
+                const vector = [];
+                for (let i = 0; i < dims; i++) {
+                    vector.push((hash[i % hash.length] / 255) * 2 - 1);
+                }
+                // Normalize
+                const norm = Math.sqrt(vector.reduce((s, v) => s + v * v, 0));
+                return vector.map((v) => v / norm);
+            });
+        },
+    };
+}
+// Placeholder for future real implementations
+export function createEmbedder(provider) {
+    // For now, always use mock. Real providers added in follow-up.
+    return createMockEmbedder(384);
+}
+//# sourceMappingURL=embedder.js.map

package/node_modules/expxagents-knowledge/dist/ingest/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/ingest/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAGjC,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,KAAK,CAAC,KAAe;YACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACxB,8CAA8C;gBAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC/D,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACrD,CAAC;gBACD,YAAY;gBACZ,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC9D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,+DAA+D;IAC/D,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC"}

package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.d.ts

@@ -0,0 +1,21 @@
+interface RawEntity {
+    name: string;
+    type: string;
+    description?: string;
+}
+interface RawRelation {
+    source: string;
+    target: string;
+    type: string;
+    description?: string;
+}
+interface ExtractionResult {
+    entities: RawEntity[];
+    relations: RawRelation[];
+}
+export declare function parseExtractionResponse(response: string): ExtractionResult;
+export declare function validateEntities(entities: RawEntity[]): RawEntity[];
+export declare function validateRelations(relations: RawRelation[]): RawRelation[];
+export declare function buildExtractionPrompt(content: string): string;
+export {};
+//# sourceMappingURL=entity-extractor.d.ts.map

package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-extractor.d.ts","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAwBA,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,WAAW;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,gBAAgB;IACxB,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,SAAS,EAAE,WAAW,EAAE,CAAC;CAC1B;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB,CAW1E;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,CAEnE;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,WAAW,EAAE,CAIzE;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAgB7D"}

package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.js

@@ -0,0 +1,54 @@
+import { z } from 'zod';
+const VALID_ENTITY_TYPES = ['technology', 'person', 'decision', 'concept', 'squad', 'agent', 'learning'];
+const VALID_RELATION_TYPES = ['produced', 'depends_on', 'learned_from', 'contradicts', 'decided', 'uses', 'replaced'];
+const RawEntitySchema = z.object({
+    name: z.string(),
+    type: z.string(),
+    description: z.string().optional(),
+});
+const RawRelationSchema = z.object({
+    source: z.string(),
+    target: z.string(),
+    type: z.string(),
+    description: z.string().optional(),
+});
+const ExtractionResponseSchema = z.object({
+    entities: z.array(RawEntitySchema).default([]),
+    relations: z.array(RawRelationSchema).default([]),
+});
+export function parseExtractionResponse(response) {
+    const codeBlockMatch = response.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+    const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : response.trim();
+    try {
+        const parsed = JSON.parse(jsonStr);
+        const validated = ExtractionResponseSchema.parse(parsed);
+        return validated;
+    }
+    catch {
+        return { entities: [], relations: [] };
+    }
+}
+export function validateEntities(entities) {
+    return entities.filter((e) => VALID_ENTITY_TYPES.includes(e.type) && e.name?.trim());
+}
+export function validateRelations(relations) {
+    return relations.filter((r) => VALID_RELATION_TYPES.includes(r.type) && r.source?.trim() && r.target?.trim());
+}
+export function buildExtractionPrompt(content) {
+    return `Given the text below, extract:
+1. Entities (name, type, description)
+2. Relations between entities (source, target, type, description)
+
+Valid entity types: ${VALID_ENTITY_TYPES.join(', ')}
+Valid relation types: ${VALID_RELATION_TYPES.join(', ')}
+
+Respond ONLY with JSON in this format:
+{
+  "entities": [{"name": "...", "type": "...", "description": "..."}],
+  "relations": [{"source": "...", "target": "...", "type": "...", "description": "..."}]
+}
+
+TEXT:
+${content}`;
+}
+//# sourceMappingURL=entity-extractor.js.map

package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-extractor.js","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,kBAAkB,GAAiB,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AACvH,MAAM,oBAAoB,GAAmB,CAAC,UAAU,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;AAEtI,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AAEH,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CAClD,CAAC,CAAC;AAoBH,MAAM,UAAU,uBAAuB,CAAC,QAAgB;IACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE5E,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,wBAAwB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AACrG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,SAAwB;IACxD,OAAO,SAAS,CAAC,MAAM,CACrB,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAoB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CACrG,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,OAAO;;;;sBAIa,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;wBAC3B,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;EASrD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.d.ts

@@ -0,0 +1,16 @@
+import type Database from 'better-sqlite3';
+export declare class ExtractionQueue {
+    private db;
+    constructor(db: Database.Database);
+    enqueue(documentId: string): void;
+    dequeue(): {
+        id: string;
+        document_id: string;
+        attempts: number;
+    } | null;
+    markComplete(jobId: string): void;
+    markFailed(jobId: string, error: string): void;
+    private cleanupStaleJobs;
+    pendingCount(): number;
+}
+//# sourceMappingURL=extraction-queue.d.ts.map

package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-queue.d.ts","sourceRoot":"","sources":["../../src/ingest/extraction-queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAK3C,qBAAa,eAAe;IACd,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ,CAAC,QAAQ;IAIzC,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOjC,OAAO,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAWvE,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IASjC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAa9C,OAAO,CAAC,gBAAgB;IAIxB,YAAY,IAAI,MAAM;CAGvB"}

package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.js

@@ -0,0 +1,49 @@
+import { nanoid } from 'nanoid';
+const MAX_RETRIES = 2;
+export class ExtractionQueue {
+    db;
+    constructor(db) {
+        this.db = db;
+        this.cleanupStaleJobs();
+    }
+    enqueue(documentId) {
+        const now = new Date().toISOString();
+        this.db.prepare('INSERT OR IGNORE INTO extraction_jobs (id, document_id, status, attempts, created_at, updated_at) VALUES (?, ?, ?, 0, ?, ?)').run(nanoid(), documentId, 'pending', now, now);
+    }
+    dequeue() {
+        const job = this.db.prepare("SELECT id, document_id, attempts FROM extraction_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1").get();
+        if (!job)
+            return null;
+        this.db.prepare("UPDATE extraction_jobs SET status = 'running', updated_at = ? WHERE id = ?")
+            .run(new Date().toISOString(), job.id);
+        return job;
+    }
+    markComplete(jobId) {
+        const now = new Date().toISOString();
+        this.db.prepare("UPDATE extraction_jobs SET status = 'complete', updated_at = ? WHERE id = ?").run(now, jobId);
+        const job = this.db.prepare('SELECT document_id FROM extraction_jobs WHERE id = ?').get(jobId);
+        if (job) {
+            this.db.prepare("UPDATE documents SET extraction_status = 'complete', updated_at = ? WHERE id = ?").run(now, job.document_id);
+        }
+    }
+    markFailed(jobId, error) {
+        const now = new Date().toISOString();
+        const job = this.db.prepare('SELECT document_id, attempts FROM extraction_jobs WHERE id = ?').get(jobId);
+        if (!job)
+            return;
+        if (job.attempts >= MAX_RETRIES) {
+            this.db.prepare("UPDATE extraction_jobs SET status = 'failed', error = ?, updated_at = ? WHERE id = ?").run(error, now, jobId);
+            this.db.prepare("UPDATE documents SET extraction_status = 'failed', updated_at = ? WHERE id = ?").run(now, job.document_id);
+        }
+        else {
+            this.db.prepare("UPDATE extraction_jobs SET status = 'pending', attempts = attempts + 1, error = ?, updated_at = ? WHERE id = ?").run(error, now, jobId);
+        }
+    }
+    cleanupStaleJobs() {
+        this.db.prepare("UPDATE extraction_jobs SET status = 'pending' WHERE status = 'running'").run();
+    }
+    pendingCount() {
+        return this.db.prepare("SELECT COUNT(*) as count FROM extraction_jobs WHERE status = 'pending'").get().count;
+    }
+}
+//# sourceMappingURL=extraction-queue.js.map

package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extraction-queue.js","sourceRoot":"","sources":["../../src/ingest/extraction-queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,MAAM,OAAO,eAAe;IACN;IAApB,YAAoB,EAAqB;QAArB,OAAE,GAAF,EAAE,CAAmB;QACvC,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAED,OAAO,CAAC,UAAkB;QACxB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,OAAO,CACb,6HAA6H,CAC9H,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB,gHAAgH,CACjH,CAAC,GAAG,EAAuE,CAAC;QAC7E,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4EAA4E,CAAC;aAC1F,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,YAAY,CAAC,KAAa;QACxB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6EAA6E,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/G,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAwC,CAAC;QACtI,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kFAAkF,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAChI,CAAC;IACH,CAAC;IAED,UAAU,CAAC,KAAa,EAAE,KAAa;QACrC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC,GAAG,CAAC,KAAK,CAA0D,CAAC;QAClK,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,IAAI,GAAG,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAC;YAChC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sFAAsF,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YAC/H,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gFAAgF,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAC9H,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gHAAgH,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3J,CAAC;IACH,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wEAAwE,CAAC,CAAC,GAAG,EAAE,CAAC;IAClG,CAAC;IAED,YAAY;QACV,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wEAAwE,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;IACtI,CAAC;CACF"}

package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.d.ts

@@ -0,0 +1,9 @@
+import type { PdfConfig, PdfMetadata } from '../types.js';
+interface PdfExtractionResult {
+    content: string;
+    metadata: PdfMetadata;
+}
+export declare function extractPdf(filePath: string, config: PdfConfig): Promise<PdfExtractionResult>;
+export declare function terminateOcrWorker(): Promise<void>;
+export {};
+//# sourceMappingURL=pdf-extractor.d.ts.map

package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extractor.d.ts","sourceRoot":"","sources":["../../src/ingest/pdf-extractor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1D,UAAU,mBAAmB;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,WAAW,CAAC;CACvB;AAID,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,GAChB,OAAO,CAAC,mBAAmB,CAAC,CA6E9B;AAqCD,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CASxD"}

package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.js

@@ -0,0 +1,116 @@
+import fs from 'node:fs';
+let ocrWorker = null;
+export async function extractPdf(filePath, config) {
+    const data = new Uint8Array(fs.readFileSync(filePath));
+    const pdfjsLib = await import('pdfjs-dist/legacy/build/pdf.mjs');
+    let doc;
+    try {
+        doc = await pdfjsLib.getDocument({ data, useSystemFonts: true }).promise;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        if (message.includes('password') || message.includes('encrypted')) {
+            throw new Error(`Password-protected PDF: ${filePath}`);
+        }
+        throw new Error(`Failed to parse PDF ${filePath}: ${message}`);
+    }
+    const pageCount = doc.numPages;
+    if (pageCount === 0) {
+        throw new Error(`Zero-page PDF: ${filePath}`);
+    }
+    const pagesToProcess = Math.min(pageCount, config.max_pages);
+    const truncated = pagesToProcess < pageCount;
+    const pageTexts = [];
+    const ocrPages = [];
+    for (let i = 1; i <= pagesToProcess; i++) {
+        const page = await doc.getPage(i);
+        const textContent = await page.getTextContent();
+        const text = textContent.items
+            .map(item => ('str' in item ? item.str : ''))
+            .join(' ')
+            .trim();
+        if (text.length >= config.ocr_threshold) {
+            pageTexts.push(text);
+        }
+        else if (config.ocr_enabled) {
+            try {
+                const ocrText = await ocrPage(page, config);
+                pageTexts.push(ocrText || text);
+                if (ocrText)
+                    ocrPages.push(i);
+            }
+            catch {
+                pageTexts.push(text);
+            }
+        }
+        else {
+            pageTexts.push(text);
+        }
+        page.cleanup();
+    }
+    let content;
+    if (pageTexts.length <= 1) {
+        content = pageTexts[0] ?? '';
+    }
+    else {
+        content = pageTexts
+            .map((text, i) => (i > 0 ? `[PAGE ${i + 1}]\n\n${text}` : text))
+            .join('\n\n');
+    }
+    const extraction_method = ocrPages.length === 0
+        ? 'native'
+        : ocrPages.length === pagesToProcess
+            ? 'ocr'
+            : 'mixed';
+    return {
+        content,
+        metadata: {
+            page_count: pageCount,
+            ocr_pages: ocrPages,
+            languages: ocrPages.length > 0 ? config.ocr_languages : [],
+            extraction_method,
+            ...(truncated ? { truncated: true } : {}),
+        },
+    };
+}
+async function ocrPage(page, config) {
+    try {
+        const Tesseract = await import('tesseract.js');
+        if (!ocrWorker) {
+            ocrWorker = await Tesseract.createWorker(config.ocr_languages.join('+'));
+        }
+        const worker = ocrWorker;
+        const viewport = page.getViewport({ scale: 2.0 });
+        // Dynamic import for optional native dependency — canvas is only needed for OCR
+        const { createCanvas } = await import(/* webpackIgnore: true */ 'canvas');
+        const canvas = createCanvas(viewport.width, viewport.height);
+        const context = canvas.getContext('2d');
+        await page.render({
+            canvasContext: context,
+            viewport,
+        }).promise;
+        const imageBuffer = canvas.toBuffer('image/png');
+        const result = await worker.recognize(imageBuffer);
+        return result.data.text.trim();
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        if (message.includes('Cannot find module') || message.includes('MODULE_NOT_FOUND')) {
+            console.warn('[pdf-extractor] OCR dependencies not available — skipping OCR');
+            return '';
+        }
+        throw err;
+    }
+}
+export async function terminateOcrWorker() {
+    if (ocrWorker) {
+        try {
+            await ocrWorker.terminate();
+        }
+        catch {
+            // Ignore termination errors
+        }
+        ocrWorker = null;
+    }
+}
+//# sourceMappingURL=pdf-extractor.js.map