@wanshi-kg/wanshi 0.1.0

package/dist/core/processor/readers/EmailReader.js

@@ -0,0 +1,259 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmailReader = void 0;
+const fs = __importStar(require("fs"));
+const mailparser_1 = require("mailparser");
+const FileReader_1 = require("./FileReader");
+const turnPacking_1 = require("./transcript/turnPacking");
+/**
+ * html-to-text profile for email bodies — rescued from the kg-mail-assistant
+ * prototype's `MailListener`: preserve document structure (headings, lists,
+ * blockquotes, tables) while dropping boilerplate (script/style/nav/footer/ads).
+ */
+const EMAIL_HTML_OPTIONS = {
+    wordwrap: false,
+    selectors: [
+        // uppercase:false — preserve original case so entity names aren't SHOUTED.
+        { selector: "h1", format: "heading", options: { uppercase: false, leadingLineBreaks: 2, trailingLineBreaks: 1 } },
+        { selector: "h2", format: "heading", options: { uppercase: false, leadingLineBreaks: 2, trailingLineBreaks: 1 } },
+        { selector: "h3", format: "heading", options: { uppercase: false, leadingLineBreaks: 2, trailingLineBreaks: 1 } },
+        { selector: "h4", format: "heading", options: { uppercase: false, leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "p", format: "paragraph", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "br", format: "lineBreak" },
+        { selector: "ul", format: "unorderedList", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "ol", format: "orderedList", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "li", format: "listItem", options: { leadingLineBreaks: 1 } },
+        { selector: "blockquote", format: "blockquote", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "table", format: "table" },
+        { selector: "a", options: { ignoreHref: true } },
+        { selector: "img", format: "skip" },
+        { selector: "script", format: "skip" },
+        { selector: "style", format: "skip" },
+        { selector: "nav", format: "skip" },
+        { selector: "header", format: "skip" },
+        { selector: "footer", format: "skip" },
+        { selector: ".navigation", format: "skip" },
+        { selector: ".sidebar", format: "skip" },
+        { selector: ".footer", format: "skip" },
+        { selector: ".header", format: "skip" },
+        { selector: ".menu", format: "skip" },
+        { selector: ".advertisement", format: "skip" },
+        { selector: ".ads", format: "skip" },
+    ],
+};
+/**
+ * Reads email (`.eml`, `.mbox`) into **size-packed** chunks by treating each
+ * message as a conversational turn — the same path as transcripts/chat exports.
+ * Sender → `speaker`, the `Date:` header → `occurredAt` (the observation's
+ * bitemporal `validAt`), so an email thread becomes a provenance-rich
+ * conversation graph with no email-specific logic leaking downstream.
+ *
+ * A `.mbox` is split into its messages (`From ` envelope lines); messages from
+ * different threads (`References`/`In-Reply-To` root) are tagged with distinct
+ * `conversation` ids so `packTurns` never packs two threads into one chunk
+ * (KG-10). HTML-only bodies are decoded with a structure-preserving,
+ * boilerplate-stripping html-to-text profile; quoted reply chains are stripped
+ * so a reply contributes its new content, not N copies of the thread.
+ *
+ * Registered before TextReader/BinaryReader so `.eml`/`.mbox` (otherwise
+ * unclaimed → skipped as binary) route here. `sourceAdapter:"email"` is stamped
+ * centrally by FileProcessor from `adapterId()`.
+ *
+ * v1 scope: `.eml` + `.mbox`. Deferred — `.msg` (binary Outlook); explicit
+ * `In-Reply-To → reply_to` edges (a packed chunk can span messages, so a single
+ * per-chunk `locator:msgid` is ambiguous too); attachment extraction.
+ */
+class EmailReader extends FileReader_1.FileReader {
+    constructor(chunker, logger, maxChunkSize, opts) {
+        super([".eml", ".mbox"], chunker, logger);
+        this.maxChunkSize = maxChunkSize;
+        this.maxMessages = opts.maxMessages;
+        this.stripQuotes = opts.stripQuotes;
+    }
+    getName() {
+        return "EmailReader";
+    }
+    adapterId() {
+        return "email";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const raw = yield fs.promises.readFile(filePath, "utf-8");
+            try {
+                const blocks = filePath.toLowerCase().endsWith(".mbox") ? this.splitMbox(raw) : [raw];
+                if (blocks.length === 0)
+                    return this.plainFallback(raw);
+                let limited = blocks;
+                if (blocks.length > this.maxMessages) {
+                    this.logger.warn(`EmailReader: ${filePath} has ${blocks.length} messages; truncating to maxMessages=${this.maxMessages}`);
+                    limited = blocks.slice(0, this.maxMessages);
+                }
+                const threadIndex = new Map();
+                const turns = [];
+                for (const block of limited) {
+                    const turn = yield this.messageToTurn(block, threadIndex);
+                    if (turn)
+                        turns.push(turn);
+                }
+                if (turns.length === 0)
+                    return this.plainFallback(raw);
+                const chunks = yield (0, turnPacking_1.packTurns)(turns, filePath, this.maxChunkSize, this.chunker);
+                return { chunks, metadata: { type: "email", source: filePath, messages: turns.length } };
+            }
+            catch (e) {
+                this.logger.warn(`EmailReader could not parse ${filePath}; falling back to plain chunking: ${e}`);
+                return this.plainFallback(raw);
+            }
+        });
+    }
+    /** Parse one RFC822 message into a conversational turn (null if empty). */
+    messageToTurn(block, threadIndex) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const parsed = yield (0, mailparser_1.simpleParser)(block);
+            // Prefer the HTML part (decoded with our boilerplate-stripping profile) over
+            // mailparser's auto-generated `text` — for HTML-only mail mailparser populates
+            // `text` itself, but without skipping nav/footer/ads, so our selectors win.
+            let body = parsed.html
+                ? (yield this.htmlToText(parsed.html)).trim()
+                : ((_a = parsed.text) !== null && _a !== void 0 ? _a : "").trim();
+            if (this.stripQuotes)
+                body = this.stripQuotedReplies(body).trim();
+            const subject = ((_b = parsed.subject) !== null && _b !== void 0 ? _b : "").trim();
+            if (!body && !subject)
+                return null;
+            const text = subject ? `Subject: ${subject}\n${body}`.trim() : body;
+            const occurredAt = parsed.date instanceof Date && !Number.isNaN(parsed.date.getTime())
+                ? parsed.date.toISOString()
+                : undefined;
+            return {
+                speaker: this.senderName(parsed.from),
+                text,
+                occurredAt,
+                conversation: this.threadIndexFor(parsed, threadIndex),
+            };
+        });
+    }
+    /** Display name from `From:` (`Name <addr>` → Name, else the address). */
+    senderName(from) {
+        var _a;
+        const v = (_a = from === null || from === void 0 ? void 0 : from.value) === null || _a === void 0 ? void 0 : _a[0];
+        if ((v === null || v === void 0 ? void 0 : v.name) && v.name.trim())
+            return v.name.trim();
+        if ((v === null || v === void 0 ? void 0 : v.address) && v.address.trim())
+            return v.address.trim();
+        if ((from === null || from === void 0 ? void 0 : from.text) && from.text.trim())
+            return from.text.trim();
+        return "unknown";
+    }
+    /**
+     * Map a message to a 0-based thread index by its conversation root
+     * (`References[0]` ?? `In-Reply-To` ?? own `Message-ID`), so distinct threads
+     * in one `.mbox` get distinct `conversation` ids and never share a chunk.
+     */
+    threadIndexFor(parsed, map) {
+        var _a, _b;
+        const refs = parsed.references;
+        const root = Array.isArray(refs)
+            ? refs[0]
+            : typeof refs === "string"
+                ? refs.trim().split(/\s+/)[0]
+                : undefined;
+        const key = ((_b = (_a = root !== null && root !== void 0 ? root : parsed.inReplyTo) !== null && _a !== void 0 ? _a : parsed.messageId) !== null && _b !== void 0 ? _b : `__${map.size}`).trim();
+        if (!map.has(key))
+            map.set(key, map.size);
+        return map.get(key);
+    }
+    /** Split an mbox into RFC822 message blocks on `From ` envelope lines. */
+    splitMbox(raw) {
+        const fromLine = /^From .+\b(?:19|20)\d{2}\b/;
+        const blocks = [];
+        let cur = [];
+        for (const line of raw.split(/\r?\n/)) {
+            if (fromLine.test(line)) {
+                if (cur.length > 0) {
+                    blocks.push(cur.join("\n"));
+                    cur = [];
+                }
+                continue; // drop the envelope line itself
+            }
+            // Unescape mbox quoting: ">From " → "From ", ">>From " → ">From ".
+            cur.push(line.replace(/^>(>*From )/, "$1"));
+        }
+        if (cur.length > 0)
+            blocks.push(cur.join("\n"));
+        return blocks.map((b) => b.trim()).filter(Boolean);
+    }
+    htmlToText(html) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { convert } = yield Promise.resolve().then(() => __importStar(require("html-to-text")));
+            return convert(html, EMAIL_HTML_OPTIONS);
+        });
+    }
+    /**
+     * Light, best-effort quote stripping: cut the body at the first quote-chain
+     * marker ("On … wrote:" / "----- Original Message -----") and drop surviving
+     * leading-`>` lines. Not a full reply parser — keeps each message's own prose.
+     */
+    stripQuotedReplies(body) {
+        const onWrote = /^\s*On\b.*\bwrote:\s*$/i;
+        const origMsg = /^\s*-{2,}\s*Original Message\s*-{2,}/i;
+        const out = [];
+        for (const line of body.split(/\r?\n/)) {
+            if (onWrote.test(line) || origMsg.test(line))
+                break;
+            out.push(line);
+        }
+        return out.filter((l) => !/^\s*>/.test(l)).join("\n");
+    }
+    plainFallback(raw) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const parts = yield this.chunker.chunk(raw);
+            return { chunks: parts.map((p) => (Object.assign({}, p))), metadata: { type: "email-fallback" } };
+        });
+    }
+}
+exports.EmailReader = EmailReader;
+//# sourceMappingURL=EmailReader.js.map

package/dist/core/processor/readers/EmailReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EmailReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/EmailReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAAsD;AAEtD,6CAA0D;AAG1D,0DAA2D;AAU3D;;;;GAIG;AACH,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ,EAAE,KAAK;IACf,SAAS,EAAE;QACT,2EAA2E;QAC3E,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QAChG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE;QACvC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACrG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACnG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,EAAE;QACzE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QAC1G,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;QACtC,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;QAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;QACnC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;QACtC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;QACrC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;QACnC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;QACtC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;QACtC,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE;QAC3C,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE;QACxC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE;QACvC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE;QACvC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;QACrC,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,EAAE;QAC9C,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;KACrC;CACF,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAa,WAAY,SAAQ,uBAAU;IAKzC,YAAY,OAAoB,EAAE,MAAc,EAAE,YAAoB,EAAE,IAAwB;QAC9F,KAAK,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACtC,CAAC;IAED,OAAO;QACL,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,SAAS;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACtF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBAExD,IAAI,OAAO,GAAG,MAAM,CAAC;gBACrB,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,gBAAgB,QAAQ,QAAQ,MAAM,CAAC,MAAM,wCAAwC,IAAI,CAAC,WAAW,EAAE,CACxG,CAAC;oBACF,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC9C,CAAC;gBAED,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;gBAC9C,MAAM,KAAK,GAAW,EAAE,CAAC;gBACzB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;oBAC1D,IAAI,IAAI;wBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7B,CAAC;gBACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBAEvD,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBACjF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3F,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,+BAA+B,QAAQ,qCAAqC,CAAC,EAAE,CAChF,CAAC;gBACF,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;KAAA;IAED,2EAA2E;IAC7D,aAAa,CAAC,KAAa,EAAE,WAAgC;;;YACzE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAY,EAAC,KAAK,CAAC,CAAC;YAEzC,6EAA6E;YAC7E,+EAA+E;YAC/E,4EAA4E;YAC5E,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI;gBACpB,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;gBAC7C,CAAC,CAAC,CAAC,MAAA,MAAM,CAAC,IAAI,mCAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,WAAW;gBAAE,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAElE,MAAM,OAAO,GAAG,CAAC,MAAA,MAAM,CAAC,OAAO,mCAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAEnC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,YAAY,OAAO,KAAK,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACpE,MAAM,UAAU,GACd,MAAM,CAAC,IAAI,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE;gBAC3B,CAAC,CAAC,SAAS,CAAC;YAEhB,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;gBACrC,IAAI;gBACJ,UAAU;gBACV,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC;aACvD,CAAC;QACJ,CAAC;KAAA;IAED,0EAA0E;IAClE,UAAU,CAAC,IAAwB;;QACzC,MAAM,CAAC,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,KAAK,0CAAG,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,IAAI,KAAI,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,OAAO,KAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,KAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5D,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;OAIG;IACK,cAAc,CAAC,MAAkB,EAAE,GAAwB;;QACjE,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAC9B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACT,CAAC,CAAC,OAAO,IAAI,KAAK,QAAQ;gBACxB,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC7B,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,GAAG,GAAG,CAAC,MAAA,MAAA,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,MAAM,CAAC,SAAS,mCAAI,MAAM,CAAC,SAAS,mCAAI,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IACvB,CAAC;IAED,0EAA0E;IAClE,SAAS,CAAC,GAAW;QAC3B,MAAM,QAAQ,GAAG,4BAA4B,CAAC;QAC9C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,GAAG,GAAa,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC5B,GAAG,GAAG,EAAE,CAAC;gBACX,CAAC;gBACD,SAAS,CAAC,gCAAgC;YAC5C,CAAC;YACD,mEAAmE;YACnE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IAEa,UAAU,CAAC,IAAY;;YACnC,MAAM,EAAE,OAAO,EAAE,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,OAAO,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;QAC3C,CAAC;KAAA;IAED;;;;OAIG;IACK,kBAAkB,CAAC,IAAY;QACrC,MAAM,OAAO,GAAG,yBAAyB,CAAC;QAC1C,MAAM,OAAO,GAAG,uCAAuC,CAAC;QACxD,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,MAAM;YACpD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;IAEa,aAAa,CAAC,GAAW;;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5C,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,CAAC;QACxF,CAAC;KAAA;CACF;AAxJD,kCAwJC"}

package/dist/core/processor/readers/EpubReader.js

@@ -0,0 +1,175 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EpubReader = void 0;
+const path = __importStar(require("path"));
+const adm_zip_1 = __importDefault(require("adm-zip"));
+const cheerio = __importStar(require("cheerio"));
+const FileReader_1 = require("./FileReader");
+/** html-to-text profile for EPUB chapter XHTML — keep structure, drop chrome. */
+const EPUB_HTML_OPTIONS = {
+    wordwrap: false,
+    selectors: [
+        { selector: "h1", format: "heading", options: { uppercase: false, leadingLineBreaks: 2, trailingLineBreaks: 1 } },
+        { selector: "h2", format: "heading", options: { uppercase: false, leadingLineBreaks: 2, trailingLineBreaks: 1 } },
+        { selector: "h3", format: "heading", options: { uppercase: false, leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "p", format: "paragraph", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "ul", format: "unorderedList", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "ol", format: "orderedList", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "li", format: "listItem" },
+        { selector: "blockquote", format: "blockquote", options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
+        { selector: "a", options: { ignoreHref: true } },
+        { selector: "img", format: "skip" },
+        { selector: "script", format: "skip" },
+        { selector: "style", format: "skip" },
+        { selector: "nav", format: "skip" },
+    ],
+};
+/**
+ * Reads EPUB (`.epub`) into **chapter-boundary** chunks. An EPUB is a ZIP of
+ * XHTML documents ordered by the OPF spine; this reader unzips (adm-zip), reads
+ * the spine (cheerio xmlMode — the `GrobidClient` precedent, no new XML dep),
+ * converts each chapter's XHTML to text (html-to-text), and chunks **each
+ * chapter separately** (re-indexed across all) so a chunk never spans two
+ * chapters. `sourceAdapter:"epub"` is stamped centrally from `adapterId()`.
+ *
+ * v1 deferred: TOC/nav-driven chapter titles, embedded images, EPUB3 nav specifics.
+ */
+class EpubReader extends FileReader_1.FileReader {
+    constructor(chunker, logger, maxChunkSize) {
+        super([".epub"], chunker, logger);
+        this.maxChunkSize = maxChunkSize;
+    }
+    getName() {
+        return "EpubReader";
+    }
+    adapterId() {
+        return "epub";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const zip = new adm_zip_1.default(filePath);
+                const opfPath = this.findOpfPath(zip);
+                if (!opfPath) {
+                    this.logger.warn(`EpubReader: no OPF rootfile in ${filePath}`);
+                    return { chunks: [], metadata: { type: "epub-error" } };
+                }
+                const chapters = yield this.readChapters(zip, opfPath);
+                if (chapters.length === 0)
+                    return { chunks: [], metadata: { type: "epub-empty" } };
+                // Chunk each chapter independently, then re-index so no chunk spans two.
+                const all = [];
+                for (const ch of chapters) {
+                    const text = (ch.title ? `# ${ch.title}\n\n` : "") + ch.text;
+                    if (!text.trim())
+                        continue;
+                    for (const p of yield this.chunker.chunk(text))
+                        all.push(Object.assign({}, p));
+                }
+                all.forEach((c, i) => {
+                    c.index = i + 1;
+                    c.totalChunks = all.length;
+                });
+                return { chunks: all, metadata: { type: "epub", chapters: chapters.length } };
+            }
+            catch (e) {
+                this.logger.warn(`EpubReader could not parse ${filePath}: ${e}`);
+                return { chunks: [], metadata: { type: "epub-error" } };
+            }
+        });
+    }
+    /** META-INF/container.xml → the OPF package path. */
+    findOpfPath(zip) {
+        const container = zip.getEntry("META-INF/container.xml");
+        if (!container)
+            return undefined;
+        const $ = cheerio.load(zip.readAsText(container), { xmlMode: true });
+        return $("rootfile").first().attr("full-path") || undefined;
+    }
+    /** OPF manifest+spine → ordered chapter {title, text}. */
+    readChapters(zip, opfPath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const $ = cheerio.load(zip.readAsText(opfPath), { xmlMode: true });
+            const hrefById = new Map();
+            $("manifest item").each((_, el) => {
+                const id = $(el).attr("id");
+                const href = $(el).attr("href");
+                if (id && href)
+                    hrefById.set(id, href);
+            });
+            const opfDir = path.posix.dirname(opfPath.split(path.sep).join("/"));
+            const chapters = [];
+            const spine = $("spine itemref").toArray();
+            for (const el of spine) {
+                const href = hrefById.get($(el).attr("idref") || "");
+                if (!href)
+                    continue;
+                const entryName = opfDir === "." ? href : `${opfDir}/${href}`;
+                const entry = (_a = zip.getEntry(entryName)) !== null && _a !== void 0 ? _a : zip.getEntry(decodeURIComponent(entryName));
+                if (!entry)
+                    continue;
+                const xhtml = zip.readAsText(entry);
+                const { title, text } = yield this.xhtmlToChapter(xhtml);
+                if (text.trim())
+                    chapters.push({ title, text });
+            }
+            return chapters;
+        });
+    }
+    xhtmlToChapter(xhtml) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const $ = cheerio.load(xhtml);
+            const title = ($("title").first().text() || $("h1").first().text() || "").trim() || undefined;
+            const { convert } = yield Promise.resolve().then(() => __importStar(require("html-to-text")));
+            return { title, text: convert(xhtml, EPUB_HTML_OPTIONS) };
+        });
+    }
+}
+exports.EpubReader = EpubReader;
+//# sourceMappingURL=EpubReader.js.map

package/dist/core/processor/readers/EpubReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EpubReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/EpubReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAC7B,sDAA6B;AAC7B,iDAAmC;AAEnC,6CAAuE;AAIvE,iFAAiF;AACjF,MAAM,iBAAiB,GAAsB;IAC3C,QAAQ,EAAE,KAAK;IACf,SAAS,EAAE;QACT,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACjH,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QAChG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACrG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QACnG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE;QACtC,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;QAC1G,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;QAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;QACnC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;QACtC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;QACrC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;KACpC;CACF,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAa,UAAW,SAAQ,uBAAU;IAGxC,YAAY,OAAoB,EAAE,MAAc,EAAE,YAAoB;QACpE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,OAAO;QACL,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,SAAS;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,iBAAM,CAAC,QAAQ,CAAC,CAAC;gBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACtC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,QAAQ,EAAE,CAAC,CAAC;oBAC/D,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,CAAC;gBAC1D,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACvD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,CAAC;gBAEnF,yEAAyE;gBACzE,MAAM,GAAG,GAAkB,EAAE,CAAC;gBAC9B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;oBAC7D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;wBAAE,SAAS;oBAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;wBAAE,GAAG,CAAC,IAAI,mBAAM,CAAC,EAAG,CAAC;gBACrE,CAAC;gBACD,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;oBACnB,CAAC,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;oBAChB,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC;gBAC7B,CAAC,CAAC,CAAC;gBACH,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAChF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,CAAC;YAC1D,CAAC;QACH,CAAC;KAAA;IAED,qDAAqD;IAC7C,WAAW,CAAC,GAAW;QAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;QACzD,IAAI,CAAC,SAAS;YAAE,OAAO,SAAS,CAAC;QACjC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,SAAS,CAAC;IAC9D,CAAC;IAED,0DAA0D;IAC5C,YAAY,CAAC,GAAW,EAAE,OAAe;;;YACrD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC3C,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBAChC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChC,IAAI,EAAE,IAAI,IAAI;oBAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAErE,MAAM,QAAQ,GAAuC,EAAE,CAAC;YACxD,MAAM,KAAK,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3C,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBACrD,IAAI,CAAC,IAAI;oBAAE,SAAS;gBACpB,MAAM,SAAS,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC9D,MAAM,KAAK,GAAG,MAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,mCAAI,GAAG,CAAC,QAAQ,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC;gBACrF,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBACpC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBACzD,IAAI,IAAI,CAAC,IAAI,EAAE;oBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC;KAAA;IAEa,cAAc,CAAC,KAAa;;YACxC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YAC9F,MAAM,EAAE,OAAO,EAAE,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,CAAC;QAC5D,CAAC;KAAA;CACF;AArFD,gCAqFC"}

package/dist/core/processor/readers/FileReader.js

@@ -0,0 +1,90 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileReader = void 0;
+const path = __importStar(require("path"));
+/**
+ * Abstract base class for file readers
+ * Each file type should implement its own reader
+ */
+class FileReader {
+    constructor(supportedExtensions, chunker, logger) {
+        this.logger = logger;
+        this.chunker = chunker;
+        this.supportedExtensions = supportedExtensions.map(ext => ext.toLowerCase());
+    }
+    /**
+     * Check if this reader supports the given file
+     */
+    canRead(filePath) {
+        const ext = path.extname(filePath).toLowerCase();
+        const base = path.basename(filePath).toLowerCase();
+        // Exact match on the extension (".ts") or, for extensionless / named files
+        // (Makefile, Dockerfile, LICENSE), the full basename. Never a prefix match:
+        // `startsWith` plus an empty-string entry once let TextReader claim every
+        // file (so .mp3/.wav read as UTF-8 and the ASR path was unreachable).
+        return this.supportedExtensions.some(e => e !== '' && (ext === e || base === e));
+    }
+    /**
+     * Stable, clean source-adapter id stamped onto every fact's provenance
+     * (`Observation.sourceAdapter`) so origin is queryable (ECS source-tagging).
+     * Defaults to the reader name minus the `Reader` suffix; readers with a more
+     * meaningful id (e.g. the PDF engines → "pdf:mistral") override this.
+     */
+    adapterId() {
+        return this.getName().replace(/Reader$/, "").toLowerCase();
+    }
+    /**
+     * Validate file before reading
+     */
+    validateFile(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.canRead(filePath)) {
+                throw new Error(`${this.getName()} does not support file extension: ${path.extname(filePath)}, supported extensions are ${this.supportedExtensions.join(', ')}`);
+            }
+        });
+    }
+}
+exports.FileReader = FileReader;
+//# sourceMappingURL=FileReader.js.map

package/dist/core/processor/readers/FileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/FileReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2CAA6B;AA8B7B;;;GAGG;AACH,MAAsB,UAAU;IAK9B,YAAY,mBAA6B,EAAE,OAAoB,EAAE,MAAc;QAC7E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,QAAgB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACnD,2EAA2E;QAC3E,4EAA4E;QAC5E,0EAA0E;QAC1E,sEAAsE;QACtE,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IACnF,CAAC;IAYD;;;;;OAKG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,CAAC;IAED;;OAEG;IACa,YAAY,CAAC,QAAgB;;YAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,CAAC,OAAO,EAAE,qCAAqC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChJ,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;CACF;AAtDD,gCAsDC"}

package/dist/core/processor/readers/FileReaderFactory.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileReaderFactory = void 0;
+/**
+ * Factory for creating appropriate file readers based on file type
+ */
+class FileReaderFactory {
+    constructor(logger) {
+        this.logger = logger;
+        this.readers = [];
+    }
+    /**
+     * Get appropriate reader for a file
+     * @param filePath Path to the file
+     * @returns FileReader instance or null if no reader supports the file
+     */
+    getReader(filePath) {
+        for (const reader of this.readers) {
+            if (reader.canRead(filePath)) {
+                this.logger.debug(`Using ${reader.getName()} for file: ${filePath}`);
+                return reader;
+            }
+        }
+        this.logger.warn(`No reader found for file: ${filePath}`);
+        return null;
+    }
+    /**
+     * Register a custom reader
+     * @param reader Custom FileReader implementation
+     */
+    registerReader(reader) {
+        this.readers.push(reader);
+        this.logger.info(`Registered custom reader: ${reader.getName()}`);
+    }
+    /**
+     * Get all registered readers
+     */
+    getReaders() {
+        return [...this.readers];
+    }
+    /**
+     * Check if any reader supports the file
+     */
+    canRead(filePath) {
+        return this.getReader(filePath) !== null;
+    }
+}
+exports.FileReaderFactory = FileReaderFactory;
+//# sourceMappingURL=FileReaderFactory.js.map

package/dist/core/processor/readers/FileReaderFactory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileReaderFactory.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/FileReaderFactory.ts"],"names":[],"mappings":";;;AAGA;;GAEG;AACH,MAAa,iBAAiB;IAG5B,YAAoB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAF1B,YAAO,GAAiB,EAAE,CAAC;IAEG,CAAC;IAEvC;;;;OAIG;IACH,SAAS,CAAC,QAAgB;QACxB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,OAAO,EAAE,cAAc,QAAQ,EAAE,CAAC,CAAC;gBACrE,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,MAAkB;QAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,QAAgB;QACtB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;IAC3C,CAAC;CACF;AA5CD,8CA4CC"}