doc-reader-sdk 0.0.0-stage → 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 sashi-does
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,221 @@
-# Temporary Holding Version
+# doc-reader-sdk
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[![CI](https://github.com/sashi-does/doc-reader-sdk/actions/workflows/ci.yml/badge.svg)](https://github.com/sashi-does/doc-reader-sdk/actions/workflows/ci.yml)
+[![npm version](https://img.shields.io/npm/v/doc-reader-sdk.svg)](https://www.npmjs.com/package/doc-reader-sdk)
+[![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+[![node](https://img.shields.io/node/v/doc-reader-sdk.svg)](package.json)
+
+A universal TypeScript/Node.js SDK for extracting plain text from documents — one API for PDF, Word, Excel, PowerPoint, CSV, JSON and more, with automatic type detection, streaming for large files, and graceful fallback instead of thrown exceptions.
+
+```ts
+import { readDoc } from 'doc-reader-sdk';
+
+const result = await readDoc('quarterly-report.pdf');
+console.log(result.content); // extracted text, ready for search, embeddings, an LLM prompt, etc.
+```
+
+## Contents
+
+- [Why this exists](#why-this-exists)
+- [Supported formats](#supported-formats)
+- [Installation](#installation)
+- [Usage](#usage)
+- [API reference](#api-reference)
+- [CLI](#cli)
+- [Large file handling](#large-file-handling)
+- [Type detection](#type-detection)
+- [Architecture](#architecture)
+- [Security](#security)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Why this exists
+
+Pulling text out of a document shouldn't require a different library — and a different mental model — for every file extension. doc-reader-sdk gives you one `read()` call that:
+
+- **Detects the real format** from magic bytes, not just the file extension (so a renamed `.docx` still works).
+- **Never throws on a malformed-but-plausible file.** Extractors degrade to a warning instead of an exception — you always get a result back.
+- **Chunks content meaningfully** — per PDF page, per spreadsheet sheet, per slide, per JSONL record — so downstream consumers (search indexes, LLM context windows) can work chunk-by-chunk.
+- **Streams instead of buffering** for text files over 10 MB.
+
+## Supported formats
+
+| Format | Extension(s) | Notes |
+|--------|-------------|-------|
+| PDF | `.pdf` | Page-by-page or full text; warns on scanned/image-only PDFs or parser failures instead of throwing |
+| Word | `.docx`, `.doc` | Full text via [mammoth](https://github.com/mwilliamson/mammoth.js) |
+| Excel | `.xlsx`, `.xls` | Every sheet as CSV by default |
+| PowerPoint | `.pptx`, `.ppt` | Slide-by-slide text extraction |
+| CSV / TSV | `.csv`, `.tsv` | Streamed line-by-line |
+| JSON | `.json` | Pretty-printed; falls back to raw text if invalid |
+| JSONL | `.jsonl` | One chunk per record |
+| Text / Markdown | `.txt`, `.md` | Auto-encoding detection; streams above 10 MB |
+| HTML / XML | `.html`, `.htm`, `.xml` | Raw text preserved |
+| RTF / EPUB | `.rtf`, `.epub` | Raw text extraction |
+| ZIP | `.zip` | Lists archive contents by default |
+| Unknown | any other extension | Best-effort text extraction with encoding detection |
+
+## Installation
+
+```bash
+npm install doc-reader-sdk
+```
+
+Requires Node.js 18+. The published package ships only compiled `dist/`, `README.md`, and `LICENSE` — no source, test fixtures, or dev tooling (see `files` in [package.json](package.json)).
+
+## Usage
+
+### One-shot
+
+```ts
+import { readDoc } from 'doc-reader-sdk';
+
+const result = await readDoc('report.pdf');
+console.log(result.content);   // full concatenated text
+console.log(result.chunks);    // per-page / per-sheet / per-slide chunks
+console.log(result.charCount); // total characters
+console.log(result.warnings);  // non-fatal issues (encoding fallback, scanned PDF, etc.)
+```
+
+### Reusable instance with default options
+
+```ts
+import { DocReader } from 'doc-reader-sdk';
+
+const reader = new DocReader({
+  pageByPage: true,   // PDF: extract per page
+  allSheets: true,    // XLSX: extract all sheets
+  maxChars: 100_000,  // truncate at 100k characters
+});
+
+const result = await reader.read('data.xlsx');
+```
+
+Per-call options override the instance's defaults:
+
+```ts
+await reader.read('other.xlsx', { allSheets: false }); // just this call, first sheet only
+```
+
+## API reference
+
+### `readDoc(filePath, options?): Promise<ExtractionResult>`
+
+Convenience wrapper around `new DocReader().read(filePath, options)`.
+
+### `class DocReader`
+
+```ts
+new DocReader(defaultOptions?: ExtractionOptions)
+```
+
+```ts
+reader.read(filePath: string, options?: ExtractionOptions): Promise<ExtractionResult>
+```
+
+### `ExtractionOptions`
+
+```ts
+interface ExtractionOptions {
+  maxChars?: number;         // truncate output across chunks. Default: unlimited
+  pageByPage?: boolean;      // PDF: per-page chunks. Default: false
+  allSheets?: boolean;       // XLSX: include every sheet. Default: true
+  listOnly?: boolean;        // ZIP: list entries only, don't extract. Default: true
+  encoding?: BufferEncoding; // force a text encoding instead of auto-detecting
+  streaming?: boolean;       // force streaming mode. Default: true for files > 10 MB
+  chunkSize?: number;        // stream chunk size in bytes. Default: 1 MB
+}
+```
+
+### `ExtractionResult`
+
+```ts
+interface ExtractionResult {
+  type: DocType;       // detected format, e.g. "pdf" | "docx" | "xlsx" | ...
+  content: string;     // full text, chunks joined with "\n\n"
+  chunks: DocChunk[];  // [{ index, content, source }]
+  charCount: number;   // content.length
+  truncated: boolean;  // true if maxChars cut content short
+  warnings: string[];  // non-fatal issues encountered during extraction
+}
+```
+
+### `DocChunk`
+
+```ts
+interface DocChunk {
+  index: number;
+  content: string;
+  source?: string; // "Page 3", "Sheet: Finance", "Slide 2", "Record 14", ...
+}
+```
+
+## CLI
+
+Installing the package also gives you a `doc-reader` binary:
+
+```bash
+npx doc-reader report.pdf
+npx doc-reader large.pdf --page-by-page --max-chars 50000
+npx doc-reader data.xlsx --no-all-sheets
+npx doc-reader archive.zip --list-only
+npx doc-reader notes.docx --json   # raw ExtractionResult as JSON
+npx doc-reader --help
+```
+
+Options mirror `ExtractionOptions`: `--max-chars <n>`, `--page-by-page`, `--no-all-sheets`, `--list-only`, `--json`.
+
+## Large file handling
+
+- Text-based files **> 10 MB** automatically switch to streaming (override with the `streaming` option).
+- PDFs are read into memory by `pdf-parse`; use `maxChars` to cap how much text you keep.
+- CSV and JSONL are read line-by-line via Node streams and stop early once `maxChars` is reached.
+- `maxChars` truncates across chunks, not just the first one — it cuts off exactly where the running total would exceed the limit.
+
+## Type detection
+
+Detection runs in layers, so a misnamed file still resolves correctly:
+
+1. **Extension map** — fast path for known extensions.
+2. **Magic bytes** — reads the first 8 bytes to recognize PDF, OLE2 (legacy Office), and ZIP-based containers.
+3. **ZIP refinement** — for anything that matches the generic ZIP signature, inspects internal paths (`word/`, `xl/`, `ppt/`, `OEBPS/`) to tell DOCX, XLSX, PPTX and EPUB apart.
+
+## Architecture
+
+```
+src/
+  index.ts              DocReader class + readDoc() export
+  cli.ts                CLI entry point
+  test.ts               Fixture-based smoke test runner (npm run test:smoke)
+  types/index.ts        ExtractionResult, DocChunk, ExtractionOptions, DocType
+  utils/helpers.ts       type detection, encoding, streaming
+  extractors/
+    pdf.ts               pdf-parse; catches parser failures and returns a warning
+    docx.ts              mammoth
+    xlsx.ts              xlsx (SheetJS)
+    pptx.ts              adm-zip + regex text-node extraction over DrawingML
+    csv.ts               readline streaming
+    json.ts              JSON.parse + JSONL line-by-line
+    text.ts              streaming read + iconv-lite encoding
+    zip.ts               adm-zip manifest listing
+```
+
+## Security
+
+doc-reader-sdk is designed to degrade gracefully rather than throw when a file is malformed, but parsing untrusted input (uploads, third-party documents) always carries some risk from the underlying parser libraries. See [SECURITY.md](SECURITY.md) for the current threat model, how to report a vulnerability, and known transitive-dependency advisories (`xlsx`/SheetJS has unpatched advisories on the npm registry; see that file for details and mitigations).
+
+## Contributing
+
+Bug reports, format requests, and PRs are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for the dev workflow and how to add a new format.
+
+```bash
+npm install
+npm run typecheck
+npm test           # vitest
+npm run test:smoke # runs every file in test-fixtures/ through the CLI formatter
+npm run build
+```
+
+## License
+
+[MIT](LICENSE)

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const index_js_1 = require("./index.js");
+// ─── ANSI colours (no deps) ───────────────────────────────────────────────────
+const C = {
+    reset: '\x1b[0m',
+    bold: '\x1b[1m',
+    dim: '\x1b[2m',
+    cyan: '\x1b[36m',
+    green: '\x1b[32m',
+    yellow: '\x1b[33m',
+    red: '\x1b[31m',
+    blue: '\x1b[34m',
+    gray: '\x1b[90m',
+};
+function c(color, text) {
+    return process.stdout.isTTY ? `${C[color]}${text}${C.reset}` : text;
+}
+function hr(char = '─', width = 72) {
+    return char.repeat(width);
+}
+function banner(result, filePath) {
+    const fileName = path.basename(filePath);
+    const fileSizeKB = (fs.statSync(filePath).size / 1024).toFixed(1);
+    console.log('\n' + c('cyan', hr('═')));
+    console.log(c('bold', `    ${fileName}`));
+    console.log(c('dim', `      Type: ${result.type.toUpperCase()}  |  Size: ${fileSizeKB} KB  |  Characters: ${result.charCount.toLocaleString()}  |  Chunks: ${result.chunks.length}`));
+    if (result.truncated) {
+        console.log(c('yellow', `  ⚠  Content truncated`));
+    }
+    if (result.warnings.length > 0) {
+        result.warnings.forEach(w => console.log(c('yellow', `  ⚠  ${w}`)));
+    }
+    console.log(c('cyan', hr('═')) + '\n');
+}
+function printResult(result) {
+    for (const chunk of result.chunks) {
+        if (result.chunks.length > 1) {
+            console.log(c('blue', hr('─')));
+            console.log(c('bold', `  ▸ ${chunk.source ?? `Chunk ${chunk.index + 1}`}`));
+            console.log(c('blue', hr('─')));
+        }
+        console.log(chunk.content);
+        console.log();
+    }
+}
+function usage() {
+    console.log(`
+${c('bold', 'doc-reader')} — Extract text from any document
+
+${c('bold', 'Usage:')}
+  npx tsx src/cli.ts <file> [options]
+
+${c('bold', 'Options:')}
+  --max-chars <n>     Truncate output at N characters
+  --page-by-page      (PDF) Show each page separately
+  --no-all-sheets     (XLSX) Only read the first sheet
+  --list-only         (ZIP) List archive contents only (default)
+  --json              Output raw JSON instead of pretty-print
+  --help              Show this help
+
+${c('bold', 'Examples:')}
+  npx tsx src/cli.ts report.pdf
+  npx tsx src/cli.ts data.xlsx
+  npx tsx src/cli.ts large.pdf --page-by-page --max-chars 50000
+  npx tsx src/cli.ts archive.zip --list-only
+  npx tsx src/cli.ts notes.docx --json
+`);
+}
+// ─── Parse argv ───────────────────────────────────────────────────────────────
+const args = process.argv.slice(2);
+if (args.length === 0 || args.includes('--help')) {
+    usage();
+    process.exit(0);
+}
+const filePath = args[0];
+const opts = {};
+let jsonMode = false;
+for (let i = 1; i < args.length; i++) {
+    const a = args[i];
+    if (a === '--max-chars' && args[i + 1]) {
+        opts.maxChars = parseInt(args[++i], 10);
+    }
+    else if (a === '--page-by-page') {
+        opts.pageByPage = true;
+    }
+    else if (a === '--no-all-sheets') {
+        opts.allSheets = false;
+    }
+    else if (a === '--list-only') {
+        opts.listOnly = true;
+    }
+    else if (a === '--json') {
+        jsonMode = true;
+    }
+}
+if (!fs.existsSync(filePath)) {
+    console.error(c('red', `✗ File not found: ${filePath}`));
+    process.exit(1);
+}
+// ─── Run ──────────────────────────────────────────────────────────────────────
+(async () => {
+    try {
+        const reader = new index_js_1.DocReader(opts);
+        const result = await reader.read(filePath);
+        if (jsonMode) {
+            console.log(JSON.stringify(result, null, 2));
+        }
+        else {
+            banner(result, filePath);
+            printResult(result);
+            console.log(c('gray', `\nDone. ${result.charCount.toLocaleString()} characters extracted from ${result.chunks.length} chunk(s).\n`));
+        }
+    }
+    catch (err) {
+        console.error(c('red', `✗ Error: ${err.message}`));
+        if (process.env.DEBUG)
+            console.error(err);
+        process.exit(1);
+    }
+})();
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2CAA6B;AAC7B,uCAAyB;AACzB,yCAAuC;AAGvC,iFAAiF;AACjF,MAAM,CAAC,GAAG;IACR,KAAK,EAAG,SAAS;IACjB,IAAI,EAAI,SAAS;IACjB,GAAG,EAAK,SAAS;IACjB,IAAI,EAAI,UAAU;IAClB,KAAK,EAAG,UAAU;IAClB,MAAM,EAAE,UAAU;IAClB,GAAG,EAAK,UAAU;IAClB,IAAI,EAAI,UAAU;IAClB,IAAI,EAAI,UAAU;CACnB,CAAC;AAEF,SAAS,CAAC,CAAC,KAAqB,EAAE,IAAY;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACtE,CAAC;AAED,SAAS,EAAE,CAAC,IAAI,GAAG,GAAG,EAAE,KAAK,GAAG,EAAE;IAChC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,MAAM,CAAC,MAAwB,EAAE,QAAgB;IACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAElE,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAG,eAAe,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,UAAU,uBAAuB,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,gBAAgB,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvL,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAAC,MAAwB;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC,MAAM,IAAI,SAAS,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YAC5E,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3B,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;AACH,CAAC;AAED,SAAS,KAAK;IACZ,OAAO,CAAC,GAAG,CAAC;EACZ,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC;;EAEvB,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC;;;EAGnB,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC;;;;;;;;EAQrB,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC;;;;;;CAMvB,CAAC,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IAAC,KAAK,EAAE,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAC,CAAC;AAE/E,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;AACzB,MAAM,IAAI,GAAsB,EAAE,CAAC;AACnC,IAAI,QAAQ,GAAG,KAAK,CAAC;AAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,IAAI,CAAC,KAAK,aAAa,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1C,CAAC;SAAM,IAAI,CAAC,KAAK,gBAAgB,EAAE,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;SAAM,IAAI,CAAC,KAAK,iBAAiB,EAAE,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IACzB,CAAC;SAAM,IAAI,CAAC,KAAK,aAAa,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;SAAM,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC1B,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC7B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,qBAAqB,QAAQ,EAAE,CAAC,CAAC,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,iFAAiF;AACjF,CAAC,KAAK,IAAI,EAAE;IACV,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,oBAAS,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE3C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACzB,WAAW,CAAC,MAAM,CAAC,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,WAAW,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,8BAA8B,MAAM,CAAC,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC,CAAC;QACvI,CAAC;IACH,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,YAAa,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,EAAE,CAAC"}

package/dist/extractors/csv.d.ts

@@ -0,0 +1,6 @@
+import type { DocChunk, ExtractionOptions } from '../types/index.js';
+export declare function extractCsv(filePath: string, sep: string, opts: ExtractionOptions): Promise<{
+    chunks: DocChunk[];
+    warnings: string[];
+}>;
+//# sourceMappingURL=csv.d.ts.map

package/dist/extractors/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/extractors/csv.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAErE,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC;IAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA0BrD"}

package/dist/extractors/csv.js

@@ -0,0 +1,62 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractCsv = extractCsv;
+const fs = __importStar(require("fs"));
+const readline = __importStar(require("readline"));
+async function extractCsv(filePath, sep, opts) {
+    const warnings = [];
+    const lines = [];
+    const rl = readline.createInterface({
+        input: fs.createReadStream(filePath, { encoding: (opts.encoding ?? 'utf-8') }),
+        crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+        lines.push(line);
+        // Yield chunks to avoid memory blowout on giant CSVs
+        if (opts.maxChars && lines.join('\n').length > opts.maxChars) {
+            warnings.push(`CSV truncated at ${lines.length} rows due to maxChars limit.`);
+            break;
+        }
+    }
+    const header = lines[0] ?? '';
+    const cols = header.split(sep).length;
+    const content = lines.join('\n');
+    return {
+        chunks: [{ index: 0, content, source: `${lines.length} rows × ${cols} columns` }],
+        warnings,
+    };
+}
+//# sourceMappingURL=csv.js.map

package/dist/extractors/csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/extractors/csv.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,gCA8BC;AAlCD,uCAAyB;AACzB,mDAAqC;AAG9B,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,GAAW,EACX,IAAuB;IAEvB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAmB,EAAE,CAAC;QAChG,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,qDAAqD;QACrD,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CAAC,oBAAoB,KAAK,CAAC,MAAM,8BAA8B,CAAC,CAAC;YAC9E,MAAM;QACR,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEjC,OAAO;QACL,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,WAAW,IAAI,UAAU,EAAE,CAAC;QACjF,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/extractors/docx.d.ts

@@ -0,0 +1,6 @@
+import type { DocChunk } from '../types/index.js';
+export declare function extractDocx(filePath: string): Promise<{
+    chunks: DocChunk[];
+    warnings: string[];
+}>;
+//# sourceMappingURL=docx.d.ts.map

package/dist/extractors/docx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.d.ts","sourceRoot":"","sources":["../../src/extractors/docx.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAElD,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAiBrD"}

package/dist/extractors/docx.js

@@ -0,0 +1,58 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractDocx = extractDocx;
+const fs = __importStar(require("fs"));
+const mammoth_1 = __importDefault(require("mammoth"));
+async function extractDocx(filePath) {
+    const warnings = [];
+    const buf = fs.readFileSync(filePath);
+    const result = await mammoth_1.default.extractRawText({ buffer: buf });
+    if (result.messages.length > 0) {
+        result.messages.forEach(m => {
+            if (m.type === 'warning')
+                warnings.push(`mammoth: ${m.message}`);
+        });
+    }
+    const text = result.value.trim();
+    return {
+        chunks: [{ index: 0, content: text, source: 'Document' }],
+        warnings,
+    };
+}
+//# sourceMappingURL=docx.js.map

package/dist/extractors/docx.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx.js","sourceRoot":"","sources":["../../src/extractors/docx.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,kCAmBC;AAvBD,uCAAyB;AACzB,sDAA8B;AAGvB,KAAK,UAAU,WAAW,CAC/B,QAAgB;IAEhB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,iBAAO,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAE7D,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC1B,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;gBAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IACjC,OAAO;QACL,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;QACzD,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/extractors/json.d.ts

@@ -0,0 +1,10 @@
+import type { DocChunk, ExtractionOptions } from '../types/index.js';
+export declare function extractJson(filePath: string, opts: ExtractionOptions): Promise<{
+    chunks: DocChunk[];
+    warnings: string[];
+}>;
+export declare function extractJsonl(filePath: string, opts: ExtractionOptions): Promise<{
+    chunks: DocChunk[];
+    warnings: string[];
+}>;
+//# sourceMappingURL=json.d.ts.map

package/dist/extractors/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/extractors/json.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAErE,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC;IAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAkBrD;AAED,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC;IAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAiCrD"}