@wanshi-kg/wanshi 0.1.0

package/dist/core/processor/readers/SubtitleReader.js

@@ -0,0 +1,145 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SubtitleReader = void 0;
+const fs = __importStar(require("fs"));
+const FileReader_1 = require("./FileReader");
+const turnPacking_1 = require("./transcript/turnPacking");
+/**
+ * Reads subtitle/caption files (`.srt`, `.vtt`) into clean, **denoised** chunks:
+ * the index numbers, timecodes, and styling tags that would otherwise pollute
+ * extraction are stripped, leaving the spoken text. When the captions carry
+ * speaker info (VTT `<v Speaker>` voice tags) the cues become attributed
+ * `Turn`s via the shared `packTurns()` (same path as transcripts/email/chat);
+ * otherwise the caption text is concatenated and size-chunked as prose.
+ * `sourceAdapter:"subtitle"` is stamped centrally from `adapterId()`.
+ *
+ * NB (deviation from the data-sinks brief): a cue offset like `00:01:23` is a
+ * **media position, not wall-clock valid-time**, so it is NOT written to
+ * `occurredAt`/`validAt` (that would fabricate bitemporal data). A cue-time
+ * `locator` (à la PDF `p.<n>`) is the right home and is deferred to v2.
+ *
+ * v1 deferred: cue-time locator, multi-language tracks, karaoke timing tags.
+ */
+class SubtitleReader extends FileReader_1.FileReader {
+    constructor(chunker, logger, maxChunkSize) {
+        super([".srt", ".vtt"], chunker, logger);
+        this.maxChunkSize = maxChunkSize;
+    }
+    getName() {
+        return "SubtitleReader";
+    }
+    adapterId() {
+        return "subtitle";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const raw = yield fs.promises.readFile(filePath, "utf-8");
+            try {
+                const cues = this.parseCues(raw);
+                if (cues.length === 0)
+                    return this.plainFallback(raw);
+                const format = filePath.toLowerCase().endsWith(".vtt") ? "vtt" : "srt";
+                // Speaker-attributed (VTT <v>) → the turn path; else denoised prose.
+                if (cues.some((c) => c.speaker)) {
+                    const turns = cues.map((c) => ({ speaker: c.speaker || "speaker", text: c.text, conversation: 0 }));
+                    const chunks = yield (0, turnPacking_1.packTurns)(turns, filePath, this.maxChunkSize, this.chunker);
+                    return { chunks, metadata: { type: "subtitle", format, cues: cues.length } };
+                }
+                const parts = yield this.chunker.chunk(cues.map((c) => c.text).join(" "));
+                return { chunks: parts.map((p) => (Object.assign({}, p))), metadata: { type: "subtitle", format, cues: cues.length } };
+            }
+            catch (e) {
+                this.logger.warn(`SubtitleReader could not parse ${filePath}; falling back to plain chunking: ${e}`);
+                return this.plainFallback(raw);
+            }
+        });
+    }
+    /** SRT blocks and VTT cues → cleaned cue text (+ speaker), consecutive dups dropped. */
+    parseCues(raw) {
+        const blocks = raw.replace(/^/, "").split(/\r?\n\r?\n+/);
+        const cues = [];
+        let prev = "";
+        for (const block of blocks) {
+            const trimmed = block.trim();
+            if (!trimmed || /^WEBVTT/.test(trimmed) || /^(NOTE|STYLE|REGION)\b/.test(trimmed))
+                continue;
+            const lines = trimmed.split(/\r?\n/);
+            const tcIdx = lines.findIndex((l) => l.includes("-->"));
+            if (tcIdx === -1)
+                continue; // header / stray block, not a cue
+            const textLines = lines.slice(tcIdx + 1);
+            if (textLines.length === 0)
+                continue;
+            const { text, speaker } = this.cleanCue(textLines.join("\n"));
+            if (!text || text === prev)
+                continue; // drop empties + rolling-caption dups
+            prev = text;
+            cues.push({ text, speaker });
+        }
+        return cues;
+    }
+    /** Capture a VTT `<v Speaker>` voice tag, then strip all caption markup. */
+    cleanCue(raw) {
+        const v = raw.match(/<v(?:\.[^ >]+)?\s+([^>]+)>/i);
+        const speaker = v ? v[1].trim() : undefined;
+        const text = raw
+            .replace(/<[^>]+>/g, "") // <v>, <i>, <b>, <c.class>, inline <00:00:01.000> timestamps
+            .replace(/&nbsp;/gi, " ")
+            .replace(/&amp;/gi, "&")
+            .replace(/&lt;/gi, "<")
+            .replace(/&gt;/gi, ">")
+            .replace(/[ \t]+/g, " ")
+            .trim();
+        return { text, speaker };
+    }
+    plainFallback(raw) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const parts = yield this.chunker.chunk(raw);
+            return { chunks: parts.map((p) => (Object.assign({}, p))), metadata: { type: "subtitle-fallback" } };
+        });
+    }
+}
+exports.SubtitleReader = SubtitleReader;
+//# sourceMappingURL=SubtitleReader.js.map

package/dist/core/processor/readers/SubtitleReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SubtitleReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/SubtitleReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,6CAA0D;AAG1D,0DAA2D;AAO3D;;;;;;;;;;;;;;;GAeG;AACH,MAAa,cAAe,SAAQ,uBAAU;IAG5C,YAAY,OAAoB,EAAE,MAAc,EAAE,YAAoB;QACpE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,OAAO;QACL,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS;QACP,OAAO,UAAU,CAAC;IACpB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBAEvE,qEAAqE;gBACrE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAW,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC5G,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAS,EAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;oBACjF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC/E,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1E,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YAC7G,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,QAAQ,qCAAqC,CAAC,EAAE,CAAC,CAAC;gBACrG,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;KAAA;IAED,wFAAwF;IAChF,SAAS,CAAC,GAAW;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAU,EAAE,CAAC;QACvB,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAE,SAAS;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,SAAS,CAAC,kCAAkC;YAC9D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YACzC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACrC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI;gBAAE,SAAS,CAAC,sCAAsC;YAC5E,IAAI,GAAG,IAAI,CAAC;YACZ,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IACpE,QAAQ,CAAC,GAAW;QAC1B,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG;aACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,6DAA6D;aACrF,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,IAAI,EAAE,CAAC;QACV,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;IAEa,aAAa,CAAC,GAAW;;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5C,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,CAAC;QAC3F,CAAC;KAAA;CACF;AA7ED,wCA6EC"}

package/dist/core/processor/readers/TesseractPdfReader.js

@@ -0,0 +1,183 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TesseractPdfReader = void 0;
+const FileReader_1 = require("./FileReader");
+const promises_1 = __importDefault(require("fs/promises"));
+/**
+ * PDF reader backed by Tesseract OCR — the **light-local** rung of the engine
+ * ladder, for hardware with no GPU and no VLM. Pure-JS/WASM, zero system binaries:
+ * `pdf-to-png-converter` (pdf.js, no native/OS deps) rasterizes each page to a PNG
+ * buffer in memory, then `tesseract.js` (WASM) OCRs it. Claims `.pdf` only.
+ *
+ * OCR is slow, so per-page text caches to a `<pdf>.tesseract.json` sidecar reused
+ * when newer than the source (re-runs never re-OCR). Any failure — render error,
+ * missing language data, empty output — **degrades to the injected pdf2json
+ * fallback** so a run never dies on one PDF and the default path stays portable.
+ * Each page is its own chunk with an ECS `locator: "p.<n>"` (PdfReader/Mistral
+ * pattern). The heavy deps are lazy-imported so a default (pdf2json) run never
+ * pays for pdfjs/WASM at startup.
+ *
+ * NB: tesseract.js fetches the language traineddata (~10-15 MB/lang) from its CDN
+ * on first use and caches it; point `langPath` at a local mirror for a fully
+ * offline weak-hardware deployment.
+ */
+class TesseractPdfReader extends FileReader_1.FileReader {
+    constructor(opts, fallback, chunker, logger, injectedDeps) {
+        super([".pdf"], chunker, logger);
+        this.opts = opts;
+        this.fallback = fallback;
+        this.injectedDeps = injectedDeps;
+    }
+    getName() {
+        return "TesseractPdfReader";
+    }
+    adapterId() {
+        return "pdf:tesseract";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            try {
+                const startTime = Date.now();
+                const { pages, cached } = yield this.ocr(filePath);
+                const usable = pages.filter((p) => p.text.trim());
+                if (usable.length === 0)
+                    throw new Error("Tesseract produced no text");
+                const chunks = [];
+                let offset = 0;
+                usable.forEach((p) => {
+                    const content = p.text;
+                    chunks.push({
+                        content,
+                        index: p.index,
+                        totalChunks: usable.length,
+                        startOffset: offset,
+                        endOffset: offset + content.length,
+                        provenance: { locator: `p.${p.index}` }, // 1-based page number (ECS locator)
+                    });
+                    offset += content.length;
+                });
+                const stats = yield promises_1.default.stat(filePath);
+                return {
+                    chunks,
+                    metadata: {
+                        type: "pdf",
+                        fileName: filePath,
+                        fileSize: stats.size,
+                        pdfEngine: "tesseract",
+                        tesseractLang: this.opts.lang,
+                        tesseractCached: cached,
+                        pageCount: usable.length,
+                        processingTimeMs: Date.now() - startTime,
+                        status: "success",
+                    },
+                };
+            }
+            catch (error) {
+                this.logger.warn(`Tesseract OCR engine failed for ${filePath} (${error.message}); falling back to pdf2json`);
+                return this.fallback.read(filePath);
+            }
+        });
+    }
+    /** Rasterize + OCR every page (or reuse the sidecar). Throws on failure. */
+    ocr(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const sidecar = `${filePath}.tesseract.json`;
+            if (yield this.sidecarIsFresh(sidecar, filePath)) {
+                this.logger.debug(`Reusing Tesseract sidecar: ${sidecar}`);
+                return { pages: JSON.parse(yield promises_1.default.readFile(sidecar, "utf-8")), cached: true };
+            }
+            const { pdfToPng, createWorker } = yield this.loadDeps();
+            const pngPages = yield pdfToPng(filePath, {
+                returnPageContent: true, // in-memory PNG buffers, no temp files on disk
+                viewportScale: this.opts.scale,
+            });
+            const worker = yield createWorker(this.opts.lang, this.opts.oem, Object.assign({}, (this.opts.langPath ? { langPath: this.opts.langPath } : {})));
+            try {
+                if (typeof this.opts.psm === "number") {
+                    yield worker.setParameters({ tessedit_pageseg_mode: this.opts.psm });
+                }
+                const pages = [];
+                for (const pg of pngPages) {
+                    if (!pg.content)
+                        continue;
+                    const { data } = yield worker.recognize(pg.content);
+                    pages.push({ index: pg.pageNumber, text: (_a = data.text) !== null && _a !== void 0 ? _a : "" });
+                }
+                yield promises_1.default.writeFile(sidecar, JSON.stringify(pages), "utf-8");
+                return { pages, cached: false };
+            }
+            finally {
+                yield worker.terminate().catch(() => undefined);
+            }
+        });
+    }
+    /** Lazy-load the heavy deps (or use the injected test doubles). */
+    loadDeps() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c, _d;
+            const pdfToPng = (_b = (_a = this.injectedDeps) === null || _a === void 0 ? void 0 : _a.pdfToPng) !== null && _b !== void 0 ? _b : (yield Promise.resolve().then(() => __importStar(require("pdf-to-png-converter")))).pdfToPng;
+            const createWorker = (_d = (_c = this.injectedDeps) === null || _c === void 0 ? void 0 : _c.createWorker) !== null && _d !== void 0 ? _d : (yield Promise.resolve().then(() => __importStar(require("tesseract.js")))).createWorker;
+            return { pdfToPng, createWorker };
+        });
+    }
+    sidecarIsFresh(sidecar, filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const [s, a] = yield Promise.all([promises_1.default.stat(sidecar), promises_1.default.stat(filePath)]);
+                return s.mtimeMs >= a.mtimeMs;
+            }
+            catch (_a) {
+                return false;
+            }
+        });
+    }
+}
+exports.TesseractPdfReader = TesseractPdfReader;
+//# sourceMappingURL=TesseractPdfReader.js.map

package/dist/core/processor/readers/TesseractPdfReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TesseractPdfReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/TesseractPdfReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAuE;AACvE,2DAA6B;AAsC7B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,kBAAmB,SAAQ,uBAAU;IAChD,YACmB,IAAsB,EACtB,QAAoB,EACrC,OAAoB,EACpB,MAAc,EACG,YAAqC;QAEtD,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QANhB,SAAI,GAAJ,IAAI,CAAkB;QACtB,aAAQ,GAAR,QAAQ,CAAY;QAGpB,iBAAY,GAAZ,YAAY,CAAyB;IAGxD,CAAC;IAED,OAAO;QACL,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,SAAS;QACP,OAAO,eAAe,CAAC;IACzB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAClD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;gBAEvE,MAAM,MAAM,GAAkB,EAAE,CAAC;gBACjC,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;oBACnB,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO;wBACP,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,WAAW,EAAE,MAAM,CAAC,MAAM;wBAC1B,WAAW,EAAE,MAAM;wBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;wBAClC,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,oCAAoC;qBAC9E,CAAC,CAAC;oBACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,OAAO;oBACL,MAAM;oBACN,QAAQ,EAAE;wBACR,IAAI,EAAE,KAAK;wBACX,QAAQ,EAAE,QAAQ;wBAClB,QAAQ,EAAE,KAAK,CAAC,IAAI;wBACpB,SAAS,EAAE,WAAW;wBACtB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;wBAC7B,eAAe,EAAE,MAAM;wBACvB,SAAS,EAAE,MAAM,CAAC,MAAM;wBACxB,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACxC,MAAM,EAAE,SAAS;qBAClB;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,mCAAmC,QAAQ,KAAK,KAAK,CAAC,OAAO,6BAA6B,CAC3F,CAAC;gBACF,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;KAAA;IAED,4EAA4E;IAC9D,GAAG,CAAC,QAAgB;;;YAChC,MAAM,OAAO,GAAG,GAAG,QAAQ,iBAAiB,CAAC;YAC7C,IAAI,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;gBAC3D,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAc,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC/F,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEzD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE;gBACxC,iBAAiB,EAAE,IAAI,EAAE,+CAA+C;gBACxE,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;aAC/B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,oBAC1D,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAC/D,CAAC;YACH,IAAI,CAAC;gBACH,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACtC,MAAM,MAAM,CAAC,aAAa,CAAC,EAAE,qBAAqB,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACvE,CAAC;gBACD,MAAM,KAAK,GAAc,EAAE,CAAC;gBAC5B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;oBAC1B,IAAI,CAAC,EAAE,CAAC,OAAO;wBAAE,SAAS;oBAC1B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;oBACpD,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,MAAA,IAAI,CAAC,IAAI,mCAAI,EAAE,EAAE,CAAC,CAAC;gBAC9D,CAAC;gBACD,MAAM,kBAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAClC,CAAC;oBAAS,CAAC;gBACT,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;KAAA;IAED,mEAAmE;IACrD,QAAQ;;;YACpB,MAAM,QAAQ,GACZ,MAAA,MAAA,IAAI,CAAC,YAAY,0CAAE,QAAQ,mCAAK,CAAC,wDAAa,sBAAsB,GAAC,CAAC,CAAC,QAAgB,CAAC;YAC1F,MAAM,YAAY,GAChB,MAAA,MAAA,IAAI,CAAC,YAAY,0CAAE,YAAY,mCAAK,CAAC,wDAAa,cAAc,GAAC,CAAC,CAAC,YAAoB,CAAC;YAC1F,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;QACpC,CAAC;KAAA;IAEa,cAAc,CAAC,OAAe,EAAE,QAAgB;;YAC5D,IAAI,CAAC;gBACH,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,kBAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxE,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC;YAChC,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KAAA;CACF;AArHD,gDAqHC"}

package/dist/core/processor/readers/TextReader.js

@@ -0,0 +1,129 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextReader = void 0;
+const fs = __importStar(require("fs"));
+const FileReader_1 = require("./FileReader");
+/**
+ * Reader for plain text files
+ */
+class TextReader extends FileReader_1.FileReader {
+    constructor(chunker, logger) {
+        super([
+            '.txt', '.rst', '.asciidoc', '.csv',
+            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+            '.py', '.pyw', '.pyi', '.pyc', '.pyd', '.pyo',
+            '.java', '.class', '.jar',
+            '.c', '.cpp', '.cc', '.cxx', '.h', '.hpp', '.hxx',
+            '.cs', '.vb', '.fs', '.fsx', '.fsi',
+            '.rb', '.rake', '.gemspec',
+            '.go', '.mod', '.sum',
+            '.rs', '.toml',
+            '.swift', '.kt', '.kts',
+            '.scala', '.sbt',
+            '.r', '.R', '.rmd', '.Rmd',
+            '.php', '.phtml',
+            '.l', '.log', '.lisp', '.lua',
+            '.vim', '.vimrc',
+            '.pl', '.pm', '.pod',
+            '.css', '.scss', '.sass', '.less',
+            '.html', '.htm', '.xhtml',
+            '.xml', '.xsl', '.xslt',
+            '.json', '.jsonl', '.geojson',
+            '.yaml', '.yml',
+            '.toml', '.ini', '.cfg', '.conf', '.config',
+            '.env', '.env.example', '.env.local', '.env.development', '.env.production',
+            '.sh', '.bash', '.zsh', '.fish', '.ksh',
+            '.ps1', '.psm1', '.psd1',
+            '.bat', '.cmd',
+            '.sql', '.pgsql', '.mysql',
+            '.dockerfile', '.dockerignore',
+            '.gitignore', '.gitattributes', '.gitmodules',
+            '.editorconfig', '.eslintrc', '.prettierrc',
+            '.npmrc', '.yarnrc', '.nvmrc',
+            // Common extensionless text files (matched by basename now that the
+            // universal '' entry is gone). Keep this list curated — genuinely
+            // unknown / extensionless files fall through to BinaryReader (skipped).
+            'LICENSE', 'LICENCE', 'README', 'CHANGELOG', 'AUTHORS', 'NOTICE', 'COPYING',
+            'Dockerfile', 'Containerfile',
+            'Makefile', 'makefile', 'GNUmakefile',
+            'Rakefile', 'Gemfile', 'Guardfile',
+            'Gruntfile', 'gulpfile',
+            'webpack.config', 'rollup.config', 'vite.config',
+            'tsconfig', 'jsconfig', 'package.json', 'package-lock.json',
+            'pom.xml', 'build.gradle', 'build.sbt',
+            'requirements.txt', 'setup.py', 'setup.cfg', 'pyproject.toml',
+            'Cargo.toml', 'Cargo.lock',
+            'go.mod', 'go.sum',
+            '.proto', '.graphql', '.gql'
+        ], chunker, logger);
+    }
+    getName() {
+        return 'TextReader';
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            try {
+                this.logger.debug(`Reading text file: ${filePath}`);
+                const content = yield fs.promises.readFile(filePath, 'utf-8');
+                const chunks = yield this.chunker.chunk(content);
+                return {
+                    chunks,
+                    metadata: {
+                        type: 'text',
+                        encoding: 'utf-8',
+                        size: content.length
+                    }
+                };
+            }
+            catch (error) {
+                this.logger.error(`Failed to read text file ${filePath}: ${error}`);
+                throw new Error(`Failed to read text file: ${error}`);
+            }
+        });
+    }
+}
+exports.TextReader = TextReader;
+//# sourceMappingURL=TextReader.js.map

package/dist/core/processor/readers/TextReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/TextReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,6CAA0D;AAI1D;;GAEG;AACH,MAAa,UAAW,SAAQ,uBAAU;IACxC,YAAY,OAAoB,EAAE,MAAc;QAC9C,KAAK,CAAC;YACJ,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM;YACnC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;YAC5C,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;YAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM;YACzB,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM;YACjD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;YACnC,KAAK,EAAE,OAAO,EAAE,UAAU;YAC1B,KAAK,EAAE,MAAM,EAAE,MAAM;YACrB,KAAK,EAAE,OAAO;YACd,QAAQ,EAAE,KAAK,EAAE,MAAM;YACvB,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM;YAC1B,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;YAC7B,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,KAAK,EAAE,MAAM;YACpB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;YACjC,OAAO,EAAE,MAAM,EAAE,QAAQ;YACzB,MAAM,EAAE,MAAM,EAAE,OAAO;YACvB,OAAO,EAAE,QAAQ,EAAE,UAAU;YAC7B,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;YAC3C,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE,kBAAkB,EAAE,iBAAiB;YAC3E,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;YACvC,MAAM,EAAE,OAAO,EAAE,OAAO;YACxB,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,QAAQ,EAAE,QAAQ;YAC1B,aAAa,EAAE,eAAe;YAC9B,YAAY,EAAE,gBAAgB,EAAE,aAAa;YAC7C,eAAe,EAAE,WAAW,EAAE,aAAa;YAC3C,QAAQ,EAAE,SAAS,EAAE,QAAQ;YAC7B,oEAAoE;YACpE,kEAAkE;YAClE,wEAAwE;YACxE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS;YAC3E,YAAY,EAAE,eAAe;YAC7B,UAAU,EAAE,UAAU,EAAE,aAAa;YACrC,UAAU,EAAE,SAAS,EAAE,WAAW;YAClC,WAAW,EAAE,UAAU;YACvB,gBAAgB,EAAE,eAAe,EAAE,aAAa;YAChD,UAAU,EAAE,UAAU,EAAE,cAAc,EAAE,mBAAmB;YAC3D,SAAS,EAAE,cAAc,EAAE,WAAW;YACtC,kBAAkB,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB;YAC7D,YAAY,EAAE,YAAY;YAC1B,QAAQ,EAAE,QAAQ;YAClB,QAAQ,EAAE,UAAU,EAAE,MAAM;SAC7B,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACtB,CAAC;IAED,OAAO;QACL,OAAO,YAAY,CAAC;IACtB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAEjD,OAAO;oBACL,MAAM;oBACN,QAAQ,EAAE;wBACR,IAAI,EAAE,MAAM;wBACZ,QAAQ,EAAE,OAAO;wBACjB,IAAI,EAAE,OAAO,CAAC,MAAM;qBACrB;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;gBACpE,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;KAAA;CACF;AA7ED,gCA6EC"}