@wanshi-kg/wanshi 0.1.0

package/dist/core/processor/readers/JsonFileReader.js

@@ -0,0 +1,232 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonFileReader = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const FileReader_1 = require("./FileReader");
+/**
+ * Token-efficient JSON reader.
+ *
+ * Re-serializes JSON compactly (strips insignificant whitespace) and chunks on
+ * structure rather than newlines/commas: a top-level array is packed by element,
+ * an object with a dominant array (e.g. `{conversations:[…]}`) is split by that
+ * array's elements (carrying a compact header of sibling keys), and `.jsonl` is
+ * split per line. Oversized single elements recurse one level (e.g. a big
+ * conversation split by its messages). Malformed JSON falls back to raw text
+ * chunking so the reader never throws.
+ *
+ * Registered before TextReader so it claims `.json`/`.jsonl`/`.geojson`.
+ */
+class JsonFileReader extends FileReader_1.FileReader {
+    constructor(options, chunker, logger) {
+        var _a;
+        super(['.json', '.jsonl', '.geojson'], chunker, logger);
+        this.strategy = (_a = options.strategy) !== null && _a !== void 0 ? _a : 'structural';
+        this.maxChunkSize = options.maxChunkSize && options.maxChunkSize > 0
+            ? options.maxChunkSize
+            : 8000;
+    }
+    getName() {
+        return 'JsonFileReader';
+    }
+    adapterId() {
+        return 'json';
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.validateFile(filePath);
+            this.logger.debug(`Reading JSON file: ${filePath}`);
+            const raw = yield fs.promises.readFile(filePath, 'utf-8');
+            const ext = path.extname(filePath).toLowerCase();
+            if (this.strategy === 'raw') {
+                return this.rawResult(raw, 'json-raw');
+            }
+            let contents;
+            let format;
+            try {
+                if (ext === '.jsonl') {
+                    const items = raw
+                        .split('\n')
+                        .map((l) => l.trim())
+                        .filter(Boolean)
+                        .map((l) => JSON.parse(l));
+                    contents = yield this.packArray(items, '');
+                    format = 'jsonl';
+                }
+                else {
+                    contents = yield this.buildChunks(JSON.parse(raw));
+                    format = 'json';
+                }
+            }
+            catch (error) {
+                this.logger.warn(`Structured JSON parse failed for ${filePath}; falling back to raw text chunking: ${error}`);
+                return this.rawResult(raw, 'json-fallback');
+            }
+            if (contents.length === 0)
+                contents = ['']; // empty doc → one empty chunk
+            return this.toResult(contents, format, raw.length);
+        });
+    }
+    /** Split a parsed JSON value into compact, structure-aligned chunk strings. */
+    buildChunks(value) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const compact = (_a = JSON.stringify(value)) !== null && _a !== void 0 ? _a : 'null';
+            if (compact.length <= this.maxChunkSize)
+                return [compact];
+            if (Array.isArray(value)) {
+                return this.packArray(value, '');
+            }
+            if (value && typeof value === 'object') {
+                const arrayKey = this.dominantArrayKey(value);
+                if (arrayKey) {
+                    const header = this.compactHeader(value, arrayKey);
+                    return this.packArray(value[arrayKey], header, arrayKey);
+                }
+            }
+            // No array to split on — degrade to raw text splitting of the compact JSON.
+            this.logger.warn('JSON value exceeds chunk size with no array to split on; using raw text split');
+            const chunks = yield this.chunker.chunk(compact);
+            return chunks.map((c) => c.content);
+        });
+    }
+    /** Pack array elements into size-bounded chunks; recurse into oversized ones. */
+    packArray(arr, header, arrayKey) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const out = [];
+            let batch = [];
+            let batchLen = header.length + 2;
+            const flush = () => {
+                if (batch.length) {
+                    out.push(this.wrap(header, arrayKey, batch));
+                    batch = [];
+                    batchLen = header.length + 2;
+                }
+            };
+            for (const el of arr) {
+                const s = (_a = JSON.stringify(el)) !== null && _a !== void 0 ? _a : 'null';
+                if (s.length > this.maxChunkSize) {
+                    flush();
+                    // Recurse one level (e.g. a huge conversation → split by its messages).
+                    const sub = yield this.buildChunks(el);
+                    out.push(...sub);
+                    continue;
+                }
+                if (batch.length && batchLen + s.length + 1 > this.maxChunkSize)
+                    flush();
+                batch.push(el);
+                batchLen += s.length + 1;
+            }
+            flush();
+            return out;
+        });
+    }
+    /** Wrap a batch of elements back into compact JSON, restoring the array key. */
+    wrap(header, arrayKey, batch) {
+        const arr = JSON.stringify(batch);
+        if (arrayKey) {
+            const k = JSON.stringify(arrayKey);
+            return header ? `{${header},${k}:${arr}}` : `{${k}:${arr}}`;
+        }
+        return header ? `{${header},"items":${arr}}` : arr;
+    }
+    /** Compact serialization of an object's non-array sibling keys (small header). */
+    compactHeader(obj, arrayKey) {
+        const header = Object.entries(obj)
+            .filter(([k]) => k !== arrayKey)
+            .map(([k, v]) => `${JSON.stringify(k)}:${JSON.stringify(v)}`)
+            .join(',');
+        // Keep headers small so they don't dominate every chunk.
+        return header.length <= Math.floor(this.maxChunkSize / 4) ? header : '';
+    }
+    /** Pick the object property holding the largest array (by serialized size). */
+    dominantArrayKey(obj) {
+        let best;
+        let bestSize = 0;
+        for (const [k, v] of Object.entries(obj)) {
+            if (Array.isArray(v) && v.length > 0) {
+                const size = JSON.stringify(v).length;
+                if (size > bestSize) {
+                    bestSize = size;
+                    best = k;
+                }
+            }
+        }
+        return best;
+    }
+    toResult(contents, format, size) {
+        let offset = 0;
+        const chunks = contents.map((content, i) => {
+            const start = offset;
+            offset += content.length;
+            return {
+                content,
+                index: i + 1,
+                totalChunks: contents.length,
+                startOffset: start,
+                endOffset: offset,
+            };
+        });
+        this.logger.info(`Split JSON into ${chunks.length} structure-aware chunk(s)`);
+        return { chunks, metadata: { type: 'json', format, size } };
+    }
+    rawResult(raw, format) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Compact first if it parses, otherwise chunk the raw text as-is.
+            let text = raw;
+            try {
+                text = JSON.stringify(JSON.parse(raw));
+            }
+            catch (_a) {
+                /* keep raw */
+            }
+            const chunks = yield this.chunker.chunk(text);
+            return { chunks, metadata: { type: 'json', format, size: raw.length } };
+        });
+    }
+}
+exports.JsonFileReader = JsonFileReader;
+//# sourceMappingURL=JsonFileReader.js.map

package/dist/core/processor/readers/JsonFileReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonFileReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/JsonFileReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,6CAAuE;AAYvE;;;;;;;;;;;;GAYG;AACH,MAAa,cAAe,SAAQ,uBAAU;IAI5C,YAAY,OAA0B,EAAE,OAAoB,EAAE,MAAc;;QAC1E,KAAK,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,GAAG,MAAA,OAAO,CAAC,QAAQ,mCAAI,YAAY,CAAC;QACjD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC;YAClE,CAAC,CAAC,OAAO,CAAC,YAAY;YACtB,CAAC,CAAC,IAAI,CAAC;IACX,CAAC;IAED,OAAO;QACL,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAEpD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YAEjD,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;gBAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YACzC,CAAC;YAED,IAAI,QAAkB,CAAC;YACvB,IAAI,MAAc,CAAC;YACnB,IAAI,CAAC;gBACH,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACrB,MAAM,KAAK,GAAG,GAAG;yBACd,KAAK,CAAC,IAAI,CAAC;yBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;yBACpB,MAAM,CAAC,OAAO,CAAC;yBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7B,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC3C,MAAM,GAAG,OAAO,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;oBACnD,MAAM,GAAG,MAAM,CAAC;gBAClB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oCAAoC,QAAQ,wCAAwC,KAAK,EAAE,CAC5F,CAAC;gBACF,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,QAAQ,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,8BAA8B;YAC1E,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC;KAAA;IAED,+EAA+E;IACjE,WAAW,CAAC,KAAc;;;YACtC,MAAM,OAAO,GAAG,MAAA,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,mCAAI,MAAM,CAAC;YAChD,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,CAAC,OAAO,CAAC,CAAC;YAE1D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACnC,CAAC;YACD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAgC,CAAC,CAAC;gBACzE,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAgC,EAAE,QAAQ,CAAC,CAAC;oBAC9E,OAAO,IAAI,CAAC,SAAS,CAAE,KAAiC,CAAC,QAAQ,CAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACrG,CAAC;YACH,CAAC;YACD,4EAA4E;YAC5E,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,+EAA+E,CAChF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACjD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;KAAA;IAED,iFAAiF;IACnE,SAAS,CACrB,GAAc,EACd,MAAc,EACd,QAAiB;;;YAEjB,MAAM,GAAG,GAAa,EAAE,CAAC;YACzB,IAAI,KAAK,GAAc,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAEjC,MAAM,KAAK,GAAG,GAAG,EAAE;gBACjB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBACjB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;oBAC7C,KAAK,GAAG,EAAE,CAAC;oBACX,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC,CAAC;YAEF,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,MAAA,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,mCAAI,MAAM,CAAC;gBACvC,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;oBACjC,KAAK,EAAE,CAAC;oBACR,wEAAwE;oBACxE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;oBACvC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;oBACjB,SAAS;gBACX,CAAC;gBACD,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY;oBAAE,KAAK,EAAE,CAAC;gBACzE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,CAAC;YACD,KAAK,EAAE,CAAC;YACR,OAAO,GAAG,CAAC;QACb,CAAC;KAAA;IAED,gFAAgF;IACxE,IAAI,CAAC,MAAc,EAAE,QAA4B,EAAE,KAAgB;QACzE,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACnC,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAC9D,CAAC;QACD,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACrD,CAAC;IAED,kFAAkF;IAC1E,aAAa,CAAC,GAA4B,EAAE,QAAgB;QAClE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;aAC/B,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC;aAC/B,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5D,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,yDAAyD;QACzD,OAAO,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED,+EAA+E;IACvE,gBAAgB,CAAC,GAA4B;QACnD,IAAI,IAAwB,CAAC;QAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBACtC,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACpB,QAAQ,GAAG,IAAI,CAAC;oBAChB,IAAI,GAAG,CAAC,CAAC;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,QAAQ,CAAC,QAAkB,EAAE,MAAc,EAAE,IAAY;QAC/D,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,MAAM,GAAkB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;YACxD,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;YACzB,OAAO;gBACL,OAAO;gBACP,KAAK,EAAE,CAAC,GAAG,CAAC;gBACZ,WAAW,EAAE,QAAQ,CAAC,MAAM;gBAC5B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,MAAM;aAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,MAAM,2BAA2B,CAAC,CAAC;QAC9E,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;IAC9D,CAAC;IAEa,SAAS,CAAC,GAAW,EAAE,MAAc;;YACjD,kEAAkE;YAClE,IAAI,IAAI,GAAG,GAAG,CAAC;YACf,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YACzC,CAAC;YAAC,WAAM,CAAC;gBACP,cAAc;YAChB,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1E,CAAC;KAAA;CACF;AAnLD,wCAmLC"}

package/dist/core/processor/readers/JupyterReader.js

@@ -0,0 +1,178 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JupyterReader = void 0;
+const fs = __importStar(require("fs"));
+const FileReader_1 = require("./FileReader");
+const MarkdownReader_1 = require("./MarkdownReader");
+/**
+ * Reads Jupyter notebooks (`.ipynb`) **cell-type-aware**: markdown cells become
+ * narrative, code cells fenced ```` ``` ```` blocks, optionally followed by their
+ * outputs — far better than feeding the raw notebook JSON to a generic reader.
+ * `sourceAdapter:"jupyter"` is stamped centrally from `adapterId()`.
+ *
+ * Outputs and image outputs are opt-in (`readers.jupyter.includeOutputs` /
+ * `includeImages`, default off) since they often carry noise (tracebacks, repr
+ * dumps, base64 blobs). Image outputs reuse the MarkdownReader image mechanism
+ * (`![…](idx)` marker → `ChunkResult.images` via `ImageBufferResolver`).
+ *
+ * v1 deferred: cell-level provenance/locator, notebook attachments, error tracebacks.
+ */
+class JupyterReader extends FileReader_1.FileReader {
+    constructor(chunker, logger, opts) {
+        super([".ipynb"], chunker, logger);
+        this.opts = opts;
+    }
+    getName() {
+        return "JupyterReader";
+    }
+    adapterId() {
+        return "jupyter";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c, _d;
+            const raw = yield fs.promises.readFile(filePath, "utf-8");
+            try {
+                const nb = JSON.parse(raw);
+                const cells = Array.isArray(nb === null || nb === void 0 ? void 0 : nb.cells) ? nb.cells : [];
+                const lang = ((_b = (_a = nb === null || nb === void 0 ? void 0 : nb.metadata) === null || _a === void 0 ? void 0 : _a.kernelspec) === null || _b === void 0 ? void 0 : _b.language) || ((_d = (_c = nb === null || nb === void 0 ? void 0 : nb.metadata) === null || _c === void 0 ? void 0 : _c.language_info) === null || _d === void 0 ? void 0 : _d.name) || "python";
+                const images = [];
+                const blocks = [];
+                for (const cell of cells) {
+                    const src = this.srcToString(cell === null || cell === void 0 ? void 0 : cell.source);
+                    if ((cell === null || cell === void 0 ? void 0 : cell.cell_type) === "markdown") {
+                        blocks.push(src);
+                    }
+                    else if ((cell === null || cell === void 0 ? void 0 : cell.cell_type) === "code") {
+                        let block = "```" + lang + "\n" + src + "\n```";
+                        if (this.opts.includeOutputs || this.opts.includeImages)
+                            block += this.renderOutputs(cell === null || cell === void 0 ? void 0 : cell.outputs, images);
+                        blocks.push(block);
+                    }
+                    else if ((cell === null || cell === void 0 ? void 0 : cell.cell_type) === "raw") {
+                        blocks.push(src);
+                    }
+                }
+                const doc = blocks.filter((b) => b.trim()).join("\n\n");
+                if (!doc.trim())
+                    return this.plainFallback(raw);
+                const parts = yield this.chunker.chunk(doc);
+                const chunks = this.opts.includeImages && images.length
+                    ? yield this.attachImages(parts, images)
+                    : parts.map((p) => (Object.assign({}, p)));
+                return { chunks, metadata: { type: "jupyter", cells: cells.length } };
+            }
+            catch (e) {
+                this.logger.warn(`JupyterReader could not parse ${filePath}; falling back to plain chunking: ${e}`);
+                return this.plainFallback(raw);
+            }
+        });
+    }
+    /** nbformat `source`/`text` is a string or an array of line-strings. */
+    srcToString(src) {
+        return Array.isArray(src) ? src.join("") : typeof src === "string" ? src : "";
+    }
+    renderOutputs(outputs, images) {
+        var _a, _b;
+        if (!Array.isArray(outputs))
+            return "";
+        const lines = [];
+        for (const o of outputs) {
+            if ((o === null || o === void 0 ? void 0 : o.output_type) === "stream") {
+                if (this.opts.includeOutputs)
+                    lines.push(this.srcToString(o.text));
+            }
+            else if ((o === null || o === void 0 ? void 0 : o.output_type) === "execute_result" || (o === null || o === void 0 ? void 0 : o.output_type) === "display_data") {
+                const data = (_a = o === null || o === void 0 ? void 0 : o.data) !== null && _a !== void 0 ? _a : {};
+                if (this.opts.includeOutputs && data["text/plain"])
+                    lines.push(this.srcToString(data["text/plain"]));
+                if (this.opts.includeImages) {
+                    const img = (_b = data["image/png"]) !== null && _b !== void 0 ? _b : data["image/jpeg"];
+                    if (typeof img === "string") {
+                        images.push({ base64: img.replace(/\s+/g, ""), alt: "cell output" });
+                        lines.push(`![cell output](${images.length - 1})`);
+                    }
+                }
+            }
+            // output_type === "error" (tracebacks) is intentionally skipped
+        }
+        return lines.length ? "\n\nOutput:\n" + lines.join("\n") : "";
+    }
+    /** Resolve `![…](idx)` markers per chunk → image buffers (the MarkdownReader path). */
+    attachImages(parts, images) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return Promise.all(parts.map((chunk) => __awaiter(this, void 0, void 0, function* () {
+                const refs = [...chunk.content.matchAll(/!\[.*?\]\((\d+)\)/g)].map((m) => parseInt(m[1], 10));
+                const imgs = [];
+                for (const ref of refs) {
+                    if (ref >= 0 && ref < images.length) {
+                        try {
+                            imgs.push({ buffer: yield MarkdownReader_1.ImageBufferResolver.resolve(images[ref]), alt: images[ref].alt });
+                        }
+                        catch (e) {
+                            this.logger.warn(`JupyterReader could not resolve image ${ref}: ${e}`);
+                        }
+                    }
+                }
+                return {
+                    content: chunk.content,
+                    images: imgs,
+                    index: chunk.index,
+                    totalChunks: chunk.totalChunks,
+                    startOffset: chunk.startOffset,
+                    endOffset: chunk.endOffset,
+                };
+            })));
+        });
+    }
+    plainFallback(raw) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const parts = yield this.chunker.chunk(raw);
+            return { chunks: parts.map((p) => (Object.assign({}, p))), metadata: { type: "jupyter-fallback" } };
+        });
+    }
+}
+exports.JupyterReader = JupyterReader;
+//# sourceMappingURL=JupyterReader.js.map

package/dist/core/processor/readers/JupyterReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JupyterReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/JupyterReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,6CAAoF;AAGpF,qDAAsE;AAUtE;;;;;;;;;;;;GAYG;AACH,MAAa,aAAc,SAAQ,uBAAU;IAC3C,YAAY,OAAoB,EAAE,MAAc,EAAmB,IAA0B;QAC3F,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAD8B,SAAI,GAAJ,IAAI,CAAsB;IAE7F,CAAC;IAED,OAAO;QACL,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,SAAS;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IAEK,IAAI,CAAC,QAAgB;;;YACzB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3B,MAAM,KAAK,GAAU,KAAK,CAAC,OAAO,CAAC,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9D,MAAM,IAAI,GAAG,CAAA,MAAA,MAAA,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,QAAQ,0CAAE,UAAU,0CAAE,QAAQ,MAAI,MAAA,MAAA,EAAE,aAAF,EAAE,uBAAF,EAAE,CAAE,QAAQ,0CAAE,aAAa,0CAAE,IAAI,CAAA,IAAI,QAAQ,CAAC;gBACjG,MAAM,MAAM,GAAoB,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAa,EAAE,CAAC;gBAE5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,MAAM,CAAC,CAAC;oBAC3C,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,SAAS,MAAK,UAAU,EAAE,CAAC;wBACnC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACnB,CAAC;yBAAM,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,SAAS,MAAK,MAAM,EAAE,CAAC;wBACtC,IAAI,KAAK,GAAG,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,OAAO,CAAC;wBAChD,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa;4BAAE,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,OAAO,EAAE,MAAM,CAAC,CAAC;wBAC5G,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACrB,CAAC;yBAAM,IAAI,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,SAAS,MAAK,KAAK,EAAE,CAAC;wBACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;gBAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;oBAAE,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBAEhD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC5C,MAAM,MAAM,GACV,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,MAAM;oBACtC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC;oBACxC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC,CAAC;gBACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,QAAQ,qCAAqC,CAAC,EAAE,CAAC,CAAC;gBACpG,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;KAAA;IAED,wEAAwE;IAChE,WAAW,CAAC,GAAQ;QAC1B,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,CAAC;IAEO,aAAa,CAAC,OAAY,EAAE,MAAuB;;QACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,WAAW,MAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACrE,CAAC;iBAAM,IAAI,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,WAAW,MAAK,gBAAgB,IAAI,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,WAAW,MAAK,cAAc,EAAE,CAAC;gBACpF,MAAM,IAAI,GAAG,MAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,IAAI,mCAAI,EAAE,CAAC;gBAC3B,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,YAAY,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACrG,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;oBAC5B,MAAM,GAAG,GAAG,MAAA,IAAI,CAAC,WAAW,CAAC,mCAAI,IAAI,CAAC,YAAY,CAAC,CAAC;oBACpD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;wBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC;wBACrE,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;YACD,gEAAgE;QAClE,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,CAAC;IAED,uFAAuF;IACzE,YAAY,CAAC,KAAoB,EAAE,MAAuB;;YACtE,OAAO,OAAO,CAAC,GAAG,CAChB,KAAK,CAAC,GAAG,CAAC,CAAO,KAAK,EAAE,EAAE;gBACxB,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC9F,MAAM,IAAI,GAAkB,EAAE,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;wBACpC,IAAI,CAAC;4BACH,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,oCAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;wBAC9F,CAAC;wBAAC,OAAO,CAAC,EAAE,CAAC;4BACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;wBACzE,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;iBAC3B,CAAC;YACJ,CAAC,CAAA,CAAC,CACH,CAAC;QACJ,CAAC;KAAA;IAEa,aAAa,CAAC,GAAW;;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5C,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAE,CAAC;QAC1F,CAAC;KAAA;CACF;AA5GD,sCA4GC"}

package/dist/core/processor/readers/LatexReader.js

@@ -0,0 +1,176 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LatexReader = void 0;
+const fs = __importStar(require("fs"));
+const FileReader_1 = require("./FileReader");
+const CITE_CMDS = "cite|citep|citet|citeauthor|citeyear|citealt|citealp|autocite|parencite|textcite|footcite";
+const CITE_RE = new RegExp(`\\\\(?:${CITE_CMDS})\\*?\\s*(?:\\[[^\\]]*\\])*\\s*\\{([^}]*)\\}`, "g");
+/**
+ * Reader for LaTeX source (`.tex`). Two jobs:
+ *  - **Clean body** — a best-effort, regex-based de-TeX (a full parser is out of
+ *    scope; "match scale to problem"): drop comments + preamble (keeping
+ *    title/author), turn `\section` → markdown headings, unwrap text-formatting
+ *    commands, drop noise environments (figure/table/tikz/bibliography), strip
+ *    residual control sequences and unescape specials → readable prose the LLM
+ *    extracts from. Far cleaner than feeding raw `.tex` to the text reader.
+ *  - **Citations** — `\cite{}`/`\citep{}`/… keys → `metadata.references.citations`
+ *    (gated by `extractCites`, the run's `references.citations` toggle), reusing
+ *    the SAME reference pipeline as Markdown/PDF (DirectoryProcessor →
+ *    buildReferenceGraph → `cites` edges) — no new edge machinery.
+ *
+ * `sourceAdapter:"latex"` is stamped centrally from `adapterId()`.
+ * Deferred: `\ref`/`\label` intra-doc cross-refs, `\input`/`\include` assembly,
+ * sibling `.bib` title resolution, math-heavy fidelity.
+ */
+class LatexReader extends FileReader_1.FileReader {
+    constructor(chunker, logger, extractCites = false) {
+        super([".tex"], chunker, logger);
+        this.extractCites = extractCites;
+    }
+    getName() {
+        return "LatexReader";
+    }
+    adapterId() {
+        return "latex";
+    }
+    read(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const raw = yield fs.promises.readFile(filePath, "utf-8");
+            try {
+                const references = {};
+                if (this.extractCites) {
+                    const citations = this.extractTexCitations(raw);
+                    if (citations.length)
+                        references.citations = citations;
+                }
+                const body = this.detex(raw);
+                const parts = yield this.chunker.chunk(body);
+                const hasRefs = !!references.citations;
+                return {
+                    chunks: parts.map((p) => (Object.assign({}, p))),
+                    metadata: Object.assign({ type: "latex", encoding: "utf-8", size: body.length }, (hasRefs ? { references } : {})),
+                };
+            }
+            catch (e) {
+                this.logger.warn(`LatexReader could not parse ${filePath}; falling back to plain chunking: ${e}`);
+                const parts = yield this.chunker.chunk(raw);
+                return { chunks: parts.map((p) => (Object.assign({}, p))), metadata: { type: "latex-fallback" } };
+            }
+        });
+    }
+    extractTexCitations(raw) {
+        const out = [];
+        const seen = new Set();
+        let m;
+        CITE_RE.lastIndex = 0;
+        while ((m = CITE_RE.exec(raw)) !== null) {
+            for (const key of m[1].split(",").map((k) => k.trim()).filter(Boolean)) {
+                if (seen.has(key))
+                    continue;
+                seen.add(key);
+                // No id field on RawCitation — the bibkey is the citation's identifier.
+                // (A sibling .bib title lookup → richer fields is deferred.)
+                out.push({ raw: key });
+            }
+        }
+        return out;
+    }
+    /** Best-effort LaTeX → plain text. Order matters; each step is a coarse pass. */
+    detex(raw) {
+        // 1. strip line comments (a % not escaped as \%)
+        let s = raw.replace(/(^|[^\\])%.*$/gm, "$1");
+        // 2. lift title/author (they live in the preamble we're about to drop)
+        const title = this.firstBraced(s, /\\title\s*\{/);
+        const author = this.firstBraced(s, /\\author\s*\{/);
+        // 3. keep only the document body when delimited
+        const doc = s.match(/\\begin\{document\}([\s\S]*?)\\end\{document\}/);
+        const head = [title ? `# ${title}` : "", author || ""].filter(Boolean).join("\n");
+        s = (head ? head + "\n\n" : "") + (doc ? doc[1] : s);
+        // 4. drop noise environments wholesale
+        s = s.replace(/\\begin\{(figure\*?|table\*?|tikzpicture|thebibliography|tabular|align\*?|equation\*?|lstlisting|verbatim|minted)\}[\s\S]*?\\end\{\1\}/g, "");
+        // 5. sectioning → markdown headings; line breaks → newlines
+        s = s
+            .replace(/\\section\*?\s*\{([^}]*)\}/g, "\n\n# $1\n")
+            .replace(/\\subsection\*?\s*\{([^}]*)\}/g, "\n\n## $1\n")
+            .replace(/\\subsubsection\*?\s*\{([^}]*)\}/g, "\n\n### $1\n")
+            .replace(/\\paragraph\*?\s*\{([^}]*)\}/g, "\n\n**$1** ")
+            .replace(/\\\\(\[[^\]]*\])?/g, "\n");
+        // 6. drop citation/ref/label commands with their args
+        s = s.replace(new RegExp(`\\\\(?:${CITE_CMDS}|ref|eqref|pageref|cref|Cref|label)\\*?\\s*(?:\\[[^\\]]*\\])*\\s*\\{[^}]*\\}`, "g"), "");
+        // 7. drop structural/preamble commands (and any bracket/brace args)
+        s = s.replace(/\\(?:usepackage|documentclass|input|include|bibliography|bibliographystyle|newcommand|renewcommand|setlength|geometry|hypersetup|maketitle|tableofcontents|footnote)\s*(?:\[[^\]]*\])*\s*(?:\{[^{}]*\})*/g, "");
+        // 8. unwrap remaining single-arg commands (two passes for light nesting)
+        for (let i = 0; i < 2; i++) {
+            s = s.replace(/\\[a-zA-Z@]+\*?\s*\{([^{}]*)\}/g, "$1");
+        }
+        // 9. strip residual control sequences + leftover braces, unescape specials
+        s = s
+            .replace(/\\[a-zA-Z@]+\*?/g, "")
+            .replace(/\\([&%$#_{}])/g, "$1")
+            .replace(/[{}]/g, "")
+            .replace(/~/g, " ")
+            .replace(/``|''/g, '"');
+        // 10. tidy whitespace
+        return s.replace(/[ \t]{2,}/g, " ").replace(/\n{3,}/g, "\n\n").trim();
+    }
+    /** Inner text of the first `\cmd{ ... }` matched by `re` (brace-balanced). */
+    firstBraced(s, re) {
+        const m = s.match(re);
+        if (!m || m.index == null)
+            return undefined;
+        let depth = 1;
+        let i = m.index + m[0].length;
+        const start = i;
+        for (; i < s.length && depth > 0; i++) {
+            if (s[i] === "{")
+                depth++;
+            else if (s[i] === "}")
+                depth--;
+        }
+        return s.slice(start, i - 1).trim() || undefined;
+    }
+}
+exports.LatexReader = LatexReader;
+//# sourceMappingURL=LatexReader.js.map

package/dist/core/processor/readers/LatexReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LatexReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/LatexReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,6CAA0D;AAK1D,MAAM,SAAS,GAAG,2FAA2F,CAAC;AAC9G,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,UAAU,SAAS,8CAA8C,EAAE,GAAG,CAAC,CAAC;AAEnG;;;;;;;;;;;;;;;;GAgBG;AACH,MAAa,WAAY,SAAQ,uBAAU;IACzC,YAAY,OAAoB,EAAE,MAAc,EAAmB,eAAwB,KAAK;QAC9F,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QADgC,iBAAY,GAAZ,YAAY,CAAiB;IAEhG,CAAC;IAED,OAAO;QACL,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,SAAS;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,UAAU,GAAkB,EAAE,CAAC;gBACrC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,MAAM,SAAS,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;oBAChD,IAAI,SAAS,CAAC,MAAM;wBAAE,UAAU,CAAC,SAAS,GAAG,SAAS,CAAC;gBACzD,CAAC;gBACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;gBACvC,OAAO;oBACL,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC;oBACpC,QAAQ,kBACN,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,OAAO,EACjB,IAAI,EAAE,IAAI,CAAC,MAAM,IACd,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACnC;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,QAAQ,qCAAqC,CAAC,EAAE,CAAC,CAAC;gBAClG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC5C,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAM,CAAC,EAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,CAAC;YACxF,CAAC;QACH,CAAC;KAAA;IAEO,mBAAmB,CAAC,GAAW;QACrC,MAAM,GAAG,GAAkB,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,wEAAwE;gBACxE,6DAA6D;gBAC7D,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,iFAAiF;IACzE,KAAK,CAAC,GAAW;QACvB,iDAAiD;QACjD,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;QAE7C,uEAAuE;QACvE,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QAEpD,gDAAgD;QAChD,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClF,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAErD,uCAAuC;QACvC,CAAC,GAAG,CAAC,CAAC,OAAO,CACX,yIAAyI,EACzI,EAAE,CACH,CAAC;QAEF,4DAA4D;QAC5D,CAAC,GAAG,CAAC;aACF,OAAO,CAAC,6BAA6B,EAAE,YAAY,CAAC;aACpD,OAAO,CAAC,gCAAgC,EAAE,aAAa,CAAC;aACxD,OAAO,CAAC,mCAAmC,EAAE,cAAc,CAAC;aAC5D,OAAO,CAAC,+BAA+B,EAAE,aAAa,CAAC;aACvD,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;QAEvC,sDAAsD;QACtD,CAAC,GAAG,CAAC,CAAC,OAAO,CACX,IAAI,MAAM,CAAC,UAAU,SAAS,8EAA8E,EAAE,GAAG,CAAC,EAClH,EAAE,CACH,CAAC;QACF,oEAAoE;QACpE,CAAC,GAAG,CAAC,CAAC,OAAO,CACX,2MAA2M,EAC3M,EAAE,CACH,CAAC;QAEF,yEAAyE;QACzE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,iCAAiC,EAAE,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,2EAA2E;QAC3E,CAAC,GAAG,CAAC;aACF,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;aAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;aAC/B,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;aAClB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE1B,sBAAsB;QACtB,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IACxE,CAAC;IAED,8EAA8E;IACtE,WAAW,CAAC,CAAS,EAAE,EAAU;QACvC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI;YAAE,OAAO,SAAS,CAAC;QAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG;gBAAE,KAAK,EAAE,CAAC;iBACrB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG;gBAAE,KAAK,EAAE,CAAC;QACjC,CAAC;QACD,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;IACnD,CAAC;CACF;AA7HD,kCA6HC"}