@wanshi-kg/wanshi 0.1.0

package/dist/core/processor/readers/MistralOcrReader.js

@@ -0,0 +1,198 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MistralOcrReader = void 0;
+const FileReader_1 = require("./FileReader");
+const path_1 = __importDefault(require("path"));
+const promises_1 = __importDefault(require("fs/promises"));
+/**
+ * PDF reader backed by the Mistral OCR HTTP API. Native fetch (no SDK, no Python),
+ * mirroring `GrobidClient`'s injected-`fetchFn` shape so it is unit-testable.
+ * Claims `.pdf` only.
+ *
+ * Flow (per kcd's pipeline): upload file (purpose=ocr) → signed URL → `POST /v1/ocr`
+ * → `pages[].markdown` → delete the upload. Each page becomes a chunk (like
+ * PdfReader's per-page chunks). OCR costs money, so results cache to a
+ * `<pdf>.mistral.json` sidecar reused when newer than the source — re-runs never
+ * re-spend. Missing key / HTTP error **degrades to the injected pdf2json fallback**.
+ */
+class MistralOcrReader extends FileReader_1.FileReader {
+    constructor(opts, fallback, chunker, logger, fetchFn = globalThis.fetch) {
+        super([".pdf"], chunker, logger);
+        this.opts = opts;
+        this.fallback = fallback;
+        this.fetchFn = fetchFn;
+    }
+    getName() {
+        return "MistralOcrReader";
+    }
+    adapterId() {
+        return "pdf:mistral";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            if (!this.opts.apiKey) {
+                this.logger.warn(`Mistral OCR engine selected but no API key (set readers.mistral.apiKey or $MISTRAL_API_KEY); falling back to pdf2json for ${filePath}`);
+                return this.fallback.read(filePath);
+            }
+            try {
+                const startTime = Date.now();
+                const { pages, cached } = yield this.ocr(filePath);
+                const usable = pages.filter((p) => p.markdown.trim());
+                if (usable.length === 0)
+                    throw new Error("Mistral OCR returned no page text");
+                const chunks = [];
+                let offset = 0;
+                usable.forEach((p, i) => {
+                    const content = p.markdown;
+                    chunks.push({
+                        content, index: i + 1, totalChunks: usable.length,
+                        startOffset: offset, endOffset: offset + content.length,
+                        provenance: { locator: `p.${p.index + 1}` }, // 1-based page number (ECS locator)
+                    });
+                    offset += content.length;
+                });
+                const stats = yield promises_1.default.stat(filePath);
+                return {
+                    chunks,
+                    metadata: {
+                        type: "pdf",
+                        fileName: filePath,
+                        fileSize: stats.size,
+                        pdfEngine: "mistral",
+                        mistralModel: this.opts.model,
+                        mistralCached: cached,
+                        pageCount: usable.length,
+                        processingTimeMs: Date.now() - startTime,
+                        status: "success",
+                    },
+                };
+            }
+            catch (error) {
+                this.logger.warn(`Mistral OCR engine failed for ${filePath} (${error.message}); falling back to pdf2json`);
+                return this.fallback.read(filePath);
+            }
+        });
+    }
+    /** OCR the PDF (or reuse the sidecar). Throws on any API failure. */
+    ocr(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const sidecar = `${filePath}.mistral.json`;
+            if (yield this.sidecarIsFresh(sidecar, filePath)) {
+                this.logger.debug(`Reusing Mistral OCR sidecar: ${sidecar}`);
+                return { pages: JSON.parse(yield promises_1.default.readFile(sidecar, "utf-8")), cached: true };
+            }
+            const buf = yield promises_1.default.readFile(filePath);
+            const fileId = yield this.uploadFile(buf, path_1.default.basename(filePath));
+            try {
+                const url = yield this.signedUrl(fileId);
+                const pages = yield this.processOcr(url);
+                yield promises_1.default.writeFile(sidecar, JSON.stringify(pages), "utf-8");
+                return { pages, cached: false };
+            }
+            finally {
+                // Best-effort cleanup of the uploaded file (don't fail the read on this).
+                yield this.deleteFile(fileId).catch((e) => this.logger.debug(`Mistral file delete failed for ${fileId}: ${e}`));
+            }
+        });
+    }
+    uploadFile(buf, fileName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const form = new FormData();
+            form.append("purpose", "ocr");
+            form.append("file", new Blob([buf]), fileName || "doc.pdf");
+            const res = yield this.fetchFn(`${this.opts.host}/v1/files`, {
+                method: "POST",
+                headers: { Authorization: `Bearer ${this.opts.apiKey}` },
+                body: form,
+                signal: this.timeout(),
+            });
+            if (!res.ok)
+                throw new Error(`Mistral files.upload ${res.status}`);
+            const id = (_a = (yield res.json())) === null || _a === void 0 ? void 0 : _a.id;
+            if (!id)
+                throw new Error("Mistral files.upload returned no id");
+            return id;
+        });
+    }
+    signedUrl(fileId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const res = yield this.fetchFn(`${this.opts.host}/v1/files/${fileId}/url?expiry=24`, {
+                method: "GET",
+                headers: { Authorization: `Bearer ${this.opts.apiKey}`, Accept: "application/json" },
+                signal: this.timeout(),
+            });
+            if (!res.ok)
+                throw new Error(`Mistral files.get_signed_url ${res.status}`);
+            const url = (_a = (yield res.json())) === null || _a === void 0 ? void 0 : _a.url;
+            if (!url)
+                throw new Error("Mistral signed-url response had no url");
+            return url;
+        });
+    }
+    processOcr(documentUrl) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const res = yield this.fetchFn(`${this.opts.host}/v1/ocr`, {
+                method: "POST",
+                headers: { Authorization: `Bearer ${this.opts.apiKey}`, "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    model: this.opts.model,
+                    document: { type: "document_url", document_url: documentUrl },
+                    include_image_base64: false,
+                }),
+                signal: this.timeout(),
+            });
+            if (!res.ok)
+                throw new Error(`Mistral ocr.process ${res.status}`);
+            const json = yield res.json();
+            const pages = Array.isArray(json === null || json === void 0 ? void 0 : json.pages) ? json.pages : [];
+            return pages.map((p, i) => ({
+                index: typeof p.index === "number" ? p.index : i,
+                markdown: typeof p.markdown === "string" ? p.markdown : "",
+            }));
+        });
+    }
+    deleteFile(fileId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const res = yield this.fetchFn(`${this.opts.host}/v1/files/${fileId}`, {
+                method: "DELETE",
+                headers: { Authorization: `Bearer ${this.opts.apiKey}` },
+                signal: this.timeout(),
+            });
+            if (!res.ok)
+                throw new Error(`Mistral files.delete ${res.status}`);
+        });
+    }
+    timeout() {
+        return typeof AbortSignal.timeout === "function"
+            ? AbortSignal.timeout(this.opts.timeoutMs)
+            : undefined;
+    }
+    sidecarIsFresh(sidecar, filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const [s, a] = yield Promise.all([promises_1.default.stat(sidecar), promises_1.default.stat(filePath)]);
+                return s.mtimeMs >= a.mtimeMs;
+            }
+            catch (_a) {
+                return false;
+            }
+        });
+    }
+}
+exports.MistralOcrReader = MistralOcrReader;
+//# sourceMappingURL=MistralOcrReader.js.map

package/dist/core/processor/readers/MistralOcrReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MistralOcrReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/MistralOcrReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,6CAAuE;AACvE,gDAAwB;AACxB,2DAA6B;AAoB7B;;;;;;;;;;GAUG;AACH,MAAa,gBAAiB,SAAQ,uBAAU;IAC9C,YACmB,IAAoB,EACpB,QAAoB,EACrC,OAAoB,EACpB,MAAc,EACG,UAAoB,UAAkB,CAAC,KAAK;QAE7D,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QANhB,SAAI,GAAJ,IAAI,CAAgB;QACpB,aAAQ,GAAR,QAAQ,CAAY;QAGpB,YAAO,GAAP,OAAO,CAAqC;IAG/D,CAAC;IAED,OAAO;QACL,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IAED,SAAS;QACP,OAAO,aAAa,CAAC;IACvB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,6HAA6H,QAAQ,EAAE,CACxI,CAAC;gBACF,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBACtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE9E,MAAM,MAAM,GAAkB,EAAE,CAAC;gBACjC,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;oBACtB,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM;wBACjD,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;wBACvD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,EAAE,EAAE,oCAAoC;qBAClF,CAAC,CAAC;oBACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,OAAO;oBACL,MAAM;oBACN,QAAQ,EAAE;wBACR,IAAI,EAAE,KAAK;wBACX,QAAQ,EAAE,QAAQ;wBAClB,QAAQ,EAAE,KAAK,CAAC,IAAI;wBACpB,SAAS,EAAE,SAAS;wBACpB,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;wBAC7B,aAAa,EAAE,MAAM;wBACrB,SAAS,EAAE,MAAM,CAAC,MAAM;wBACxB,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACxC,MAAM,EAAE,SAAS;qBAClB;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,iCAAiC,QAAQ,KAAK,KAAK,CAAC,OAAO,6BAA6B,CACzF,CAAC;gBACF,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;KAAA;IAED,qEAAqE;IACvD,GAAG,CAAC,QAAgB;;YAChC,MAAM,OAAO,GAAG,GAAG,QAAQ,eAAe,CAAC;YAC3C,IAAI,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAC7D,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAc,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC/F,CAAC;YAED,MAAM,GAAG,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YACnE,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACzC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBACzC,MAAM,kBAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAClC,CAAC;oBAAS,CAAC;gBACT,0EAA0E;gBAC1E,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,MAAM,KAAK,CAAC,EAAE,CAAC,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;IAEa,UAAU,CAAC,GAAW,EAAE,QAAgB;;;YACpD,MAAM,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,IAAI,SAAS,CAAC,CAAC;YAC5D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,EAAE;gBAC3D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE;gBACxD,IAAI,EAAE,IAAI;gBACV,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,MAAM,EAAE,GAAG,MAAA,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,0CAAE,EAAE,CAAC;YAClC,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;KAAA;IAEa,SAAS,CAAC,MAAc;;;YACpC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,aAAa,MAAM,gBAAgB,EAAE;gBACnF,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;gBACpF,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3E,MAAM,GAAG,GAAG,MAAA,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,0CAAE,GAAG,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YACpE,OAAO,GAAG,CAAC;QACb,CAAC;KAAA;IAEa,UAAU,CAAC,WAAmB;;YAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE;gBACzD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC5F,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;oBACtB,QAAQ,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,WAAW,EAAE;oBAC7D,oBAAoB,EAAE,KAAK;iBAC5B,CAAC;gBACF,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YAClE,MAAM,IAAI,GAAQ,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,KAAK,GAAU,KAAK,CAAC,OAAO,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChD,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;aAC3D,CAAC,CAAC,CAAC;QACN,CAAC;KAAA;IAEa,UAAU,CAAC,MAAc;;YACrC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,aAAa,MAAM,EAAE,EAAE;gBACrE,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE;gBACxD,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,CAAC;KAAA;IAEO,OAAO;QACb,OAAO,OAAQ,WAAmB,CAAC,OAAO,KAAK,UAAU;YACvD,CAAC,CAAE,WAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YACnD,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC;IAEa,cAAc,CAAC,OAAe,EAAE,QAAgB;;YAC5D,IAAI,CAAC;gBACH,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,kBAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxE,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC;YAChC,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KAAA;CACF;AApKD,4CAoKC"}

package/dist/core/processor/readers/OfficeReader.js

@@ -0,0 +1,174 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OfficeReader = void 0;
+const FileReader_1 = require("./FileReader");
+const path_1 = __importDefault(require("path"));
+const officeParser = __importStar(require("officeparser"));
+const promises_1 = __importDefault(require("fs/promises"));
+/**
+ * Reader for Microsoft Office documents using officeparser
+ *
+ * Supported formats: .docx, .pptx, .xlsx, .odt, .odp, .ods
+ *
+ * NOTE: This implementation only extracts text content.
+ * Images and rich metadata are not supported by the underlying officeparser library.
+ *
+ * npm install officeparser
+ */
+class OfficeReader extends FileReader_1.FileReader {
+    constructor(chunker, logger) {
+        super([".docx", ".pptx", ".xlsx", ".odt", ".odp", ".ods"], chunker, logger);
+    }
+    getName() {
+        return "OfficeReader";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            try {
+                this.logger.debug(`Reading Office file: ${filePath}`);
+                // Get file stats for basic metadata
+                const stats = yield promises_1.default.stat(filePath);
+                const ext = path_1.default.extname(filePath).toLowerCase();
+                // Configure officeparser
+                const config = {
+                    newlineDelimiter: "\n",
+                    ignoreNotes: false, // Include notes in parsed text
+                    outputErrorToConsole: false,
+                };
+                // Extract text content using officeparser
+                const content = yield officeParser.parseOfficeAsync(filePath, config);
+                // Build metadata object
+                const metadata = {
+                    type: this.getDocumentType(ext),
+                    description: this.matchExtensionToDescription(ext),
+                    fileName: path_1.default.basename(filePath),
+                    filePath: filePath,
+                    fileSize: stats.size,
+                    createdAt: stats.birthtime.toISOString(),
+                    modifiedAt: stats.mtime.toISOString(),
+                    accessedAt: stats.atime.toISOString(),
+                    extension: ext,
+                    status: "success",
+                    // Note: officeparser doesn't provide document metadata like author, title, etc.
+                    // These would need to be extracted using format-specific libraries
+                    extractedTextLength: content.length,
+                    hasContent: content.trim().length > 0,
+                };
+                this.logger.debug(`Successfully extracted ${content.length} characters from ${filePath}`);
+                return {
+                    chunks: [
+                        {
+                            content: content,
+                            startOffset: 0,
+                            endOffset: content.length,
+                            index: 1,
+                            totalChunks: 1,
+                        },
+                    ],
+                    // images: undefined, // officeparser doesn't extract images
+                    metadata: metadata,
+                };
+            }
+            catch (error) {
+                this.logger.error(`Failed to read Office file ${filePath}: ${error.message}`);
+                // Return error result instead of throwing
+                return {
+                    chunks: [
+                        {
+                            content: "",
+                            startOffset: 0,
+                            endOffset: 0,
+                            index: 1,
+                            totalChunks: 1,
+                        },
+                    ],
+                    metadata: {
+                        type: this.getDocumentType(path_1.default.extname(filePath).toLowerCase()),
+                        description: this.matchExtensionToDescription(path_1.default.extname(filePath)),
+                        fileName: path_1.default.basename(filePath),
+                        filePath: filePath,
+                        status: "error",
+                        error: error.message,
+                        errorType: error.name,
+                    },
+                };
+            }
+        });
+    }
+    getDocumentType(ext) {
+        const types = {
+            ".docx": "word_document",
+            ".doc": "word_document_legacy",
+            ".pptx": "powerpoint_presentation",
+            ".ppt": "powerpoint_presentation_legacy",
+            ".xlsx": "excel_spreadsheet",
+            ".xls": "excel_spreadsheet_legacy",
+            ".odt": "openoffice_text",
+            ".odp": "openoffice_presentation",
+            ".ods": "openoffice_spreadsheet",
+        };
+        return types[ext] || "unknown_office_document";
+    }
+    matchExtensionToDescription(ext) {
+        const descriptions = {
+            ".docx": "Microsoft Word Document",
+            ".doc": "Microsoft Word 2003 Document",
+            ".pptx": "Microsoft PowerPoint Presentation",
+            ".ppt": "Microsoft PowerPoint 2003 Presentation",
+            ".xlsx": "Microsoft Excel Spreadsheet",
+            ".xls": "Microsoft Excel 2003 Spreadsheet",
+            ".odt": "OpenDocument Text Document",
+            ".odp": "OpenDocument Presentation",
+            ".ods": "OpenDocument Spreadsheet",
+        };
+        return descriptions[ext] || "Unknown Office Document";
+    }
+}
+exports.OfficeReader = OfficeReader;
+//# sourceMappingURL=OfficeReader.js.map

package/dist/core/processor/readers/OfficeReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OfficeReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/OfficeReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA0D;AAC1D,gDAAwB;AACxB,2DAA6C;AAC7C,2DAA6B;AAI7B;;;;;;;;;GASG;AACH,MAAa,YAAa,SAAQ,uBAAU;IAC1C,YAAY,OAAoB,EAAE,MAAc;QAC9C,KAAK,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO;QACL,OAAO,cAAc,CAAC;IACxB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAEtD,oCAAoC;gBACpC,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAM,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBAEjD,yBAAyB;gBACzB,MAAM,MAAM,GAAG;oBACb,gBAAgB,EAAE,IAAI;oBACtB,WAAW,EAAE,KAAK,EAAE,+BAA+B;oBACnD,oBAAoB,EAAE,KAAK;iBAC5B,CAAC;gBAEF,0CAA0C;gBAC1C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAEtE,wBAAwB;gBACxB,MAAM,QAAQ,GAAG;oBACf,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC;oBAC/B,WAAW,EAAE,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC;oBAClD,QAAQ,EAAE,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjC,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;oBACxC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;oBACrC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;oBACrC,SAAS,EAAE,GAAG;oBACd,MAAM,EAAE,SAAS;oBACjB,gFAAgF;oBAChF,mEAAmE;oBACnE,mBAAmB,EAAE,OAAO,CAAC,MAAM;oBACnC,UAAU,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;iBACtC,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,0BAA0B,OAAO,CAAC,MAAM,oBAAoB,QAAQ,EAAE,CACvE,CAAC;gBAEF,OAAO;oBACL,MAAM,EAAE;wBACN;4BACE,OAAO,EAAE,OAAO;4BAChB,WAAW,EAAE,CAAC;4BACd,SAAS,EAAE,OAAO,CAAC,MAAM;4BACzB,KAAK,EAAE,CAAC;4BACR,WAAW,EAAE,CAAC;yBACf;qBACF;oBACD,4DAA4D;oBAC5D,QAAQ,EAAE,QAAQ;iBACnB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8BAA8B,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAC3D,CAAC;gBAEF,0CAA0C;gBAC1C,OAAO;oBACL,MAAM,EAAE;wBACN;4BACE,OAAO,EAAE,EAAE;4BACX,WAAW,EAAE,CAAC;4BACd,SAAS,EAAE,CAAC;4BACZ,KAAK,EAAE,CAAC;4BACR,WAAW,EAAE,CAAC;yBACf;qBACF;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;wBAChE,WAAW,EAAE,IAAI,CAAC,2BAA2B,CAAC,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;wBACrE,QAAQ,EAAE,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACjC,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,OAAO;wBACf,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,SAAS,EAAE,KAAK,CAAC,IAAI;qBACtB;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;IAEO,eAAe,CAAC,GAAW;QACjC,MAAM,KAAK,GAA8B;YACvC,OAAO,EAAE,eAAe;YACxB,MAAM,EAAE,sBAAsB;YAC9B,OAAO,EAAE,yBAAyB;YAClC,MAAM,EAAE,gCAAgC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,MAAM,EAAE,0BAA0B;YAClC,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,yBAAyB;YACjC,MAAM,EAAE,wBAAwB;SACjC,CAAC;QACF,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,yBAAyB,CAAC;IACjD,CAAC;IAEO,2BAA2B,CAAC,GAAW;QAC7C,MAAM,YAAY,GAA8B;YAC9C,OAAO,EAAE,yBAAyB;YAClC,MAAM,EAAE,8BAA8B;YACtC,OAAO,EAAE,mCAAmC;YAC5C,MAAM,EAAE,wCAAwC;YAChD,OAAO,EAAE,6BAA6B;YACtC,MAAM,EAAE,kCAAkC;YAC1C,MAAM,EAAE,4BAA4B;YACpC,MAAM,EAAE,2BAA2B;YACnC,MAAM,EAAE,0BAA0B;SACnC,CAAC;QACF,OAAO,YAAY,CAAC,GAAG,CAAC,IAAI,yBAAyB,CAAC;IACxD,CAAC;CACF;AA1HD,oCA0HC"}

package/dist/core/processor/readers/PdfReader.js

@@ -0,0 +1,116 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PdfReader = void 0;
+exports.extractArxivId = extractArxivId;
+const FileReader_1 = require("./FileReader");
+const stripReferences_1 = require("./stripReferences");
+const referenceExtraction_1 = require("./referenceExtraction");
+const pdf2json_1 = __importDefault(require("pdf2json"));
+/**
+ * Extract an arXiv identifier from page text (the sidebar/footer stamp on
+ * arXiv-hosted PDFs). Pure — exported for tests.
+ */
+function extractArxivId(text) {
+    const m = text.match(/arXiv:\s*(\d{4}\.\d{4,5}(?:v\d+)?)/i);
+    return m ? m[1] : undefined;
+}
+/**
+ * Reader for PDF files
+ */
+class PdfReader extends FileReader_1.FileReader {
+    constructor(chunker, logger, stripReferences = false, extractCites = false) {
+        super([".pdf"], chunker, logger);
+        this.stripReferences = stripReferences;
+        this.extractCites = extractCites;
+    }
+    getName() {
+        return "PdfReader";
+    }
+    adapterId() {
+        return "pdf:pdf2json";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            try {
+                this.logger.debug(`Reading PDF file: ${filePath}`);
+                let { pages: content, title } = yield this.readPdfPages(filePath);
+                // Document identity is captured here, at ingest time — never extracted
+                // from body text, which is full of OTHER papers' arXiv IDs (citations).
+                const arxivId = extractArxivId(content.slice(0, 2).join("\n"));
+                // Detect the trailing bibliography once if either consumer needs it.
+                const split = this.stripReferences || this.extractCites
+                    ? (0, stripReferences_1.splitPagesAtReferences)(content)
+                    : undefined;
+                // Phase 0 citation extraction (network-free), gated by config. Drop the
+                // paper's OWN arXiv id (the host identity, not a citation).
+                let references;
+                if (this.extractCites) {
+                    const cites = (0, referenceExtraction_1.extractCitations)(split === null || split === void 0 ? void 0 : split.references, content.join("\n")).filter((c) => !arxivId || c.arxivId !== arxivId);
+                    if (cites.length)
+                        references = { citations: cites };
+                }
+                if (this.stripReferences && (split === null || split === void 0 ? void 0 : split.references)) {
+                    this.logger.info(`Quarantined trailing references section of ${filePath} ` +
+                        `(${split.references.length} chars, ${content.length - split.pages.length} page(s) dropped)`);
+                    content = split.pages;
+                }
+                const chunks = content.map((page, index) => {
+                    const startOffset = content.slice(0, index).reduce((acc, curr) => acc + curr.length, 0);
+                    return {
+                        content: page,
+                        startOffset: startOffset,
+                        endOffset: startOffset + page.length,
+                        index: index + 1,
+                        totalChunks: content.length,
+                        provenance: { locator: `p.${index + 1}` }, // 1-based page number (ECS locator)
+                    };
+                });
+                return {
+                    chunks: chunks,
+                    metadata: Object.assign(Object.assign(Object.assign({ type: "pdf", fileName: filePath }, (arxivId ? { arxivId } : {})), (title ? { title } : {})), (references ? { references } : {})),
+                };
+            }
+            catch (error) {
+                this.logger.error(`Failed to read PDF file ${filePath}: ${error}`);
+                throw new Error(`Failed to read PDF file: ${error}`);
+            }
+        });
+    }
+    readPdfPages(filePath) {
+        return new Promise((resolve, reject) => {
+            const pdfParser = new pdf2json_1.default();
+            pdfParser.on("pdfParser_dataError", (errData) => reject(errData.parserError));
+            pdfParser.on("pdfParser_dataReady", (pdfData) => {
+                var _a;
+                const pages = pdfData.Pages.map((page) => page.Texts.map((t) => t.R.map((r) => decodeURIComponent(r.T)).join("")).join("\n"));
+                const rawTitle = (_a = pdfData.Meta) === null || _a === void 0 ? void 0 : _a.Title;
+                let title;
+                if (typeof rawTitle === "string" && rawTitle.trim()) {
+                    try {
+                        title = decodeURIComponent(rawTitle).trim();
+                    }
+                    catch (_b) {
+                        title = rawTitle.trim();
+                    }
+                }
+                resolve({ pages, title });
+            });
+            pdfParser.loadPDF(filePath);
+        });
+    }
+}
+exports.PdfReader = PdfReader;
+//# sourceMappingURL=PdfReader.js.map

package/dist/core/processor/readers/PdfReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PdfReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/PdfReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAWA,wCAGC;AAZD,6CAA0D;AAC1D,uDAA2D;AAC3D,+DAAwE;AACxE,wDAAiC;AAEjC;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAAY;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC5D,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAa,SAAU,SAAQ,uBAAU;IACvC,YACE,OAAoB,EACpB,MAAc,EACG,kBAA2B,KAAK,EAChC,eAAwB,KAAK;QAE9C,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAHhB,oBAAe,GAAf,eAAe,CAAiB;QAChC,iBAAY,GAAZ,YAAY,CAAiB;IAGhD,CAAC;IAED,OAAO;QACL,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,SAAS;QACP,OAAO,cAAc,CAAC;IACxB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;gBAEnD,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAElE,uEAAuE;gBACvE,wEAAwE;gBACxE,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE/D,qEAAqE;gBACrE,MAAM,KAAK,GACT,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,YAAY;oBACvC,CAAC,CAAC,IAAA,wCAAsB,EAAC,OAAO,CAAC;oBACjC,CAAC,CAAC,SAAS,CAAC;gBAEhB,wEAAwE;gBACxE,4DAA4D;gBAC5D,IAAI,UAAqC,CAAC;gBAC1C,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,MAAM,KAAK,GAAG,IAAA,sCAAgB,EAAC,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAC1E,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO,CACzC,CAAC;oBACF,IAAI,KAAK,CAAC,MAAM;wBAAE,UAAU,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;gBACtD,CAAC;gBAED,IAAI,IAAI,CAAC,eAAe,KAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,UAAU,CAAA,EAAE,CAAC;oBAC9C,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,8CAA8C,QAAQ,GAAG;wBACvD,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,WAAW,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,mBAAmB,CAC/F,CAAC;oBACF,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC;gBACxB,CAAC;gBAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBACzC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBACxF,OAAO;wBACL,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,WAAW;wBACxB,SAAS,EAAE,WAAW,GAAG,IAAI,CAAC,MAAM;wBACpC,KAAK,EAAE,KAAK,GAAG,CAAC;wBAChB,WAAW,EAAE,OAAO,CAAC,MAAM;wBAC3B,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,KAAK,GAAG,CAAC,EAAE,EAAE,EAAE,oCAAoC;qBAChF,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,OAAO;oBACL,MAAM,EAAE,MAAM;oBACd,QAAQ,8CACN,IAAI,EAAE,KAAK,EACX,QAAQ,EAAE,QAAQ,IACf,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAC5B,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACxB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACtC;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;gBACnE,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;KAAA;IAEO,YAAY,CAAC,QAAgB;QACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,SAAS,GAAG,IAAI,kBAAS,EAAE,CAAC;YAElC,SAAS,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,OAAO,EAAE,EAAE,CAC9C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAC5B,CAAC;YAEF,SAAS,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,OAAO,EAAE,EAAE;;gBAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACvC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACjD,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;gBACF,MAAM,QAAQ,GAAG,MAAC,OAAe,CAAC,IAAI,0CAAE,KAAK,CAAC;gBAC9C,IAAI,KAAyB,CAAC;gBAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;oBACpD,IAAI,CAAC;wBACH,KAAK,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC9C,CAAC;oBAAC,WAAM,CAAC;wBACP,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AA/GD,8BA+GC"}

package/dist/core/processor/readers/RtfReader.js

@@ -0,0 +1,107 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RtfReader = void 0;
+const FileReader_1 = require("./FileReader");
+const path_1 = __importDefault(require("path"));
+const promises_1 = __importDefault(require("fs/promises"));
+const shared_1 = require("../../../shared");
+/**
+ * Reader for Rich Text Format (RTF) documents
+ *
+ * Uses the rtf-parser npm package to extract text content from RTF files.
+ * RTF documents can contain rich formatting, images, and other elements,
+ * but this reader focuses on extracting plain text content for indexing.
+ *
+ * Supported formats: .rtf
+ *
+ * npm install rtf-parser
+ */
+class RtfReader extends FileReader_1.FileReader {
+    constructor(chunker, logger) {
+        super([".rtf"], chunker, logger);
+    }
+    getName() {
+        return "RtfReader";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            try {
+                this.logger.debug(`Reading RTF file: ${filePath}`);
+                // Get file stats for basic metadata
+                const stats = yield promises_1.default.stat(filePath);
+                // Parse RTF content using rtf-parser
+                const { text, info } = yield (0, shared_1.readRtf)(filePath);
+                // Chunk the extracted text
+                const chunks = yield this.chunker.chunk(text);
+                // Convert to ChunkResult format
+                const chunkResults = chunks.map(chunk => ({
+                    content: chunk.content,
+                    startOffset: chunk.startOffset,
+                    endOffset: chunk.endOffset,
+                    index: chunk.index,
+                    totalChunks: chunk.totalChunks,
+                }));
+                // Build metadata object
+                const metadata = {
+                    type: "rtf_document",
+                    description: "Rich Text Format Document",
+                    fileName: path_1.default.basename(filePath),
+                    filePath: filePath,
+                    fileSize: stats.size,
+                    createdAt: stats.birthtime.toISOString(),
+                    modifiedAt: stats.mtime.toISOString(),
+                    accessedAt: stats.atime.toISOString(),
+                    extension: ".rtf",
+                    status: "success",
+                    extractedTextLength: text.length,
+                    hasContent: text.trim().length > 0,
+                    documentInfo: info,
+                };
+                this.logger.debug(`Successfully extracted ${text.length} characters from RTF file ${filePath}`);
+                return {
+                    chunks: chunkResults,
+                    metadata: metadata,
+                };
+            }
+            catch (error) {
+                this.logger.error(`Failed to read RTF file ${filePath}: ${error.message}`);
+                // Return error result instead of throwing
+                return {
+                    chunks: [
+                        {
+                            content: "",
+                            startOffset: 0,
+                            endOffset: 0,
+                            index: 1,
+                            totalChunks: 1,
+                        },
+                    ],
+                    metadata: {
+                        type: "rtf_document",
+                        description: "Rich Text Format Document",
+                        fileName: path_1.default.basename(filePath),
+                        filePath: filePath,
+                        status: "error",
+                        error: error.message,
+                        errorType: error.name,
+                    },
+                };
+            }
+        });
+    }
+}
+exports.RtfReader = RtfReader;
+//# sourceMappingURL=RtfReader.js.map

package/dist/core/processor/readers/RtfReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RtfReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/RtfReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,6CAA0D;AAC1D,gDAAwB;AACxB,2DAA6B;AAG7B,4CAA0C;AAE1C;;;;;;;;;;GAUG;AACH,MAAa,SAAU,SAAQ,uBAAU;IACvC,YAAY,OAAoB,EAAE,MAAc;QAC9C,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,OAAO;QACL,OAAO,WAAW,CAAC;IACrB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;gBAEnD,oCAAoC;gBACpC,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAEtC,qCAAqC;gBACrC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,gBAAO,EAAC,QAAQ,CAAC,CAAC;gBAE/C,2BAA2B;gBAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAE9C,gCAAgC;gBAChC,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACxC,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,WAAW,EAAE,KAAK,CAAC,WAAW;iBAC/B,CAAC,CAAC,CAAC;gBAEJ,wBAAwB;gBACxB,MAAM,QAAQ,GAAG;oBACf,IAAI,EAAE,cAAc;oBACpB,WAAW,EAAE,2BAA2B;oBACxC,QAAQ,EAAE,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjC,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;oBACxC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;oBACrC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;oBACrC,SAAS,EAAE,MAAM;oBACjB,MAAM,EAAE,SAAS;oBACjB,mBAAmB,EAAE,IAAI,CAAC,MAAM;oBAChC,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;oBAClC,YAAY,EAAE,IAAI;iBACnB,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,0BAA0B,IAAI,CAAC,MAAM,6BAA6B,QAAQ,EAAE,CAC7E,CAAC;gBAEF,OAAO;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,QAAQ;iBACnB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,2BAA2B,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CACxD,CAAC;gBAEF,0CAA0C;gBAC1C,OAAO;oBACL,MAAM,EAAE;wBACN;4BACE,OAAO,EAAE,EAAE;4BACX,WAAW,EAAE,CAAC;4BACd,SAAS,EAAE,CAAC;4BACZ,KAAK,EAAE,CAAC;4BACR,WAAW,EAAE,CAAC;yBACf;qBACF;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,cAAc;wBACpB,WAAW,EAAE,2BAA2B;wBACxC,QAAQ,EAAE,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACjC,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,OAAO;wBACf,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,SAAS,EAAE,KAAK,CAAC,IAAI;qBACtB;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;CACF;AAtFD,8BAsFC"}