doc-reader-sdk 0.0.0-stage → 1.0.0

package/dist/index.js

@@ -0,0 +1,181 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocReader = void 0;
+exports.readDoc = readDoc;
+const path = __importStar(require("path"));
+const helpers_js_1 = require("./utils/helpers.js");
+const pdf_js_1 = require("./extractors/pdf.js");
+const docx_js_1 = require("./extractors/docx.js");
+const xlsx_js_1 = require("./extractors/xlsx.js");
+const pptx_js_1 = require("./extractors/pptx.js");
+const csv_js_1 = require("./extractors/csv.js");
+const json_js_1 = require("./extractors/json.js");
+const text_js_1 = require("./extractors/text.js");
+const zip_js_1 = require("./extractors/zip.js");
+class DocReader {
+    constructor(defaultOptions = {}) {
+        this.defaultOptions = {
+            allSheets: true,
+            listOnly: true,
+            pageByPage: false,
+            chunkSize: 1024 * 1024, // 1 MB
+            ...defaultOptions,
+        };
+    }
+    /**
+     * Read any document file and return its full text content.
+     * Handles PDF, DOCX, XLSX, PPTX, CSV, JSON, JSONL, TXT, MD, HTML, XML, ZIP, and more.
+     */
+    async read(filePath, options = {}) {
+        const opts = { ...this.defaultOptions, ...options };
+        const absPath = path.resolve(filePath);
+        const docType = await (0, helpers_js_1.detectDocType)(absPath);
+        let chunks = [];
+        let warnings = [];
+        switch (docType) {
+            case 'pdf': {
+                const res = await (0, pdf_js_1.extractPdf)(absPath, opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'docx':
+            case 'doc': {
+                const res = await (0, docx_js_1.extractDocx)(absPath);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                if (docType === 'doc') {
+                    warnings.unshift('Legacy .doc format — results may be incomplete. Convert to .docx for best results.');
+                }
+                break;
+            }
+            case 'xlsx':
+            case 'xls': {
+                const res = await (0, xlsx_js_1.extractXlsx)(absPath, opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'pptx':
+            case 'ppt': {
+                const res = await (0, pptx_js_1.extractPptx)(absPath);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                if (docType === 'ppt') {
+                    warnings.unshift('Legacy .ppt format detected. Results may be incomplete.');
+                }
+                break;
+            }
+            case 'csv': {
+                const res = await (0, csv_js_1.extractCsv)(absPath, ',', opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'tsv': {
+                const res = await (0, csv_js_1.extractCsv)(absPath, '\t', opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'json': {
+                const res = await (0, json_js_1.extractJson)(absPath, opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'jsonl': {
+                const res = await (0, json_js_1.extractJsonl)(absPath, opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'zip': {
+                const res = await (0, zip_js_1.extractZip)(absPath, opts.listOnly !== false);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+            case 'txt':
+            case 'md':
+            case 'html':
+            case 'xml':
+            case 'rtf':
+            case 'epub':
+            case 'log':
+            case 'unknown':
+            default: {
+                const res = await (0, text_js_1.extractText)(absPath, opts);
+                chunks = res.chunks;
+                warnings = res.warnings;
+                break;
+            }
+        }
+        // Apply maxChars truncation across chunks if set
+        let truncated = false;
+        if (opts.maxChars) {
+            let remaining = opts.maxChars;
+            for (const chunk of chunks) {
+                if (remaining <= 0) {
+                    chunk.content = '';
+                }
+                else if (chunk.content.length > remaining) {
+                    chunk.content = chunk.content.slice(0, remaining) + '\n\n[… truncated …]';
+                    remaining = 0;
+                    truncated = true;
+                }
+                else {
+                    remaining -= chunk.content.length;
+                }
+            }
+        }
+        const fullContent = chunks.map(c => c.content).join('\n\n');
+        return {
+            type: docType,
+            content: fullContent,
+            chunks,
+            charCount: fullContent.length,
+            truncated,
+            warnings,
+        };
+    }
+}
+exports.DocReader = DocReader;
+/** Convenience one-shot function */
+async function readDoc(filePath, options) {
+    return new DocReader().read(filePath, options);
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJA,0BAKC;AA9JD,2CAA6B;AAC7B,mDAA6D;AAC7D,gDAAiD;AACjD,kDAAmD;AACnD,kDAAmD;AACnD,kDAAmD;AACnD,gDAAiD;AACjD,kDAAiE;AACjE,kDAAmD;AACnD,gDAAiD;AAKjD,MAAa,SAAS;IAGpB,YAAY,iBAAoC,EAAE;QAChD,IAAI,CAAC,cAAc,GAAG;YACpB,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO;YAC/B,GAAG,cAAc;SAClB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI,CACR,QAAgB,EAChB,UAA6B,EAAE;QAE/B,MAAM,IAAI,GAAsB,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,OAAO,EAAE,CAAC;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,OAAO,GAAY,MAAM,IAAA,0BAAa,EAAC,OAAO,CAAC,CAAC;QAEtD,IAAI,MAAM,GAA+B,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAa,EAAE,CAAC;QAE5B,QAAQ,OAAO,EAAE,CAAC;YAChB,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAU,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,MAAM,CAAC;YACZ,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAW,EAAC,OAAO,CAAC,CAAC;gBACvC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;oBACtB,QAAQ,CAAC,OAAO,CAAC,oFAAoF,CAAC,CAAC;gBACzG,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,MAAM,CAAC;YACZ,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAW,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,MAAM,CAAC;YACZ,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAW,EAAC,OAAO,CAAC,CAAC;gBACvC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;oBACtB,QAAQ,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC;gBAC9E,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAU,EAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;gBACjD,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAU,EAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAW,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAY,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC9C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,KAAK,CAAC,CAAC,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAU,EAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;gBAC/D,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,KAAK,KAAK,CAAC;YACX,KAAK,IAAI,CAAC;YACV,KAAK,MAAM,CAAC;YACZ,KAAK,KAAK,CAAC;YACX,KAAK,KAAK,CAAC;YACX,KAAK,MAAM,CAAC;YACZ,KAAK,KAAK,CAAC;YACX,KAAK,SAAS,CAAC;YACf,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAW,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;gBACxB,MAAM;YACR,CAAC;QACH,CAAC;QAED,iDAAiD;QACjD,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;oBACnB,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC;gBACrB,CAAC;qBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;oBAC5C,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,qBAAqB,CAAC;oBAC1E,SAAS,GAAG,CAAC,CAAC;oBACd,SAAS,GAAG,IAAI,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBACpC,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5D,OAAO;YACL,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,WAAW;YACpB,MAAM;YACN,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,SAAS;YACT,QAAQ;SACT,CAAC;IACJ,CAAC;CACF;AAxID,8BAwIC;AAED,oCAAoC;AAC7B,KAAK,UAAU,OAAO,CAC3B,QAAgB,EAChB,OAA2B;IAE3B,OAAO,IAAI,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC"}

package/dist/types/index.d.ts

@@ -0,0 +1,38 @@
+export type DocType = 'pdf' | 'docx' | 'doc' | 'xlsx' | 'xls' | 'pptx' | 'ppt' | 'csv' | 'tsv' | 'json' | 'jsonl' | 'txt' | 'md' | 'html' | 'xml' | 'rtf' | 'epub' | 'zip' | 'log' | 'unknown';
+export interface ExtractionOptions {
+    /** Max characters to return. Default: unlimited */
+    maxChars?: number;
+    /** For PDFs: extract page-by-page. Default: false */
+    pageByPage?: boolean;
+    /** For spreadsheets: include all sheets. Default: true */
+    allSheets?: boolean;
+    /** For ZIP: list files only, don't extract. Default: true */
+    listOnly?: boolean;
+    /** Encoding hint for text files. Default: auto-detect */
+    encoding?: BufferEncoding;
+    /** Stream large files in chunks instead of buffering. Default: true for >10MB */
+    streaming?: boolean;
+    /** Chunk size for streaming in bytes. Default: 1MB */
+    chunkSize?: number;
+}
+export interface DocChunk {
+    index: number;
+    content: string;
+    /** Source label: page number, sheet name, slide index, etc. */
+    source?: string;
+}
+export interface ExtractionResult {
+    /** Detected document type */
+    type: DocType;
+    /** Full concatenated content */
+    content: string;
+    /** Individual chunks (pages, sheets, slides…) */
+    chunks: DocChunk[];
+    /** Total character count */
+    charCount: number;
+    /** Whether content was truncated by maxChars */
+    truncated: boolean;
+    /** Warning messages (encoding fallback, partial extraction, etc.) */
+    warnings: string[];
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,OAAO,GACf,KAAK,GACL,MAAM,GACN,KAAK,GACL,MAAM,GACN,KAAK,GACL,MAAM,GACN,KAAK,GACL,KAAK,GACL,KAAK,GACL,MAAM,GACN,OAAO,GACP,KAAK,GACL,IAAI,GACJ,MAAM,GACN,KAAK,GACL,KAAK,GACL,MAAM,GACN,KAAK,GACL,KAAK,GACL,SAAS,CAAC;AAEd,MAAM,WAAW,iBAAiB;IAChC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,0DAA0D;IAC1D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,yDAAyD;IACzD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,iFAAiF;IACjF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,6BAA6B;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,SAAS,EAAE,OAAO,CAAC;IACnB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB"}

package/dist/types/index.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":""}

package/dist/utils/helpers.d.ts

@@ -0,0 +1,13 @@
+import type { DocType } from '../types/index.js';
+export declare function detectDocType(filePath: string): Promise<DocType>;
+export declare function readTextFile(filePath: string, hint?: BufferEncoding): Promise<{
+    text: string;
+    encoding: string;
+}>;
+export declare function streamTextFile(filePath: string, chunkSize?: number): AsyncGenerator<string>;
+export declare function truncate(text: string, maxChars?: number): {
+    text: string;
+    truncated: boolean;
+};
+export declare function fileSizeBytes(filePath: string): number;
+//# sourceMappingURL=helpers.d.ts.map

package/dist/utils/helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/utils/helpers.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AA6DjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBtE;AAID,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,cAAc,GACpB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAgB7C;AAID,wBAAuB,cAAc,CACnC,QAAQ,EAAE,MAAM,EAChB,SAAS,SAAc,GACtB,cAAc,CAAC,MAAM,CAAC,CAQxB;AAID,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,MAAM,GAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAMtC;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEtD"}

package/dist/utils/helpers.js

@@ -0,0 +1,164 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectDocType = detectDocType;
+exports.readTextFile = readTextFile;
+exports.streamTextFile = streamTextFile;
+exports.truncate = truncate;
+exports.fileSizeBytes = fileSizeBytes;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const chardet_1 = __importDefault(require("chardet"));
+const iconv_lite_1 = __importDefault(require("iconv-lite"));
+// ─── Type Detection ────────────────────────────────────────────────────────────
+const EXT_MAP = {
+    pdf: 'pdf',
+    docx: 'docx',
+    doc: 'doc',
+    xlsx: 'xlsx',
+    xls: 'xls',
+    pptx: 'pptx',
+    ppt: 'ppt',
+    csv: 'csv',
+    tsv: 'tsv',
+    json: 'json',
+    jsonl: 'jsonl',
+    txt: 'txt',
+    md: 'md',
+    markdown: 'md',
+    html: 'html',
+    htm: 'html',
+    xml: 'xml',
+    rtf: 'rtf',
+    epub: 'epub',
+    zip: 'zip',
+    log: 'log',
+};
+/** Magic-byte signatures for format sniffing (first 8 bytes) */
+const MAGIC = [
+    { bytes: [0x25, 0x50, 0x44, 0x46], type: 'pdf' }, // %PDF
+    { bytes: [0x50, 0x4b, 0x03, 0x04], type: 'zip' }, // PK (zip, docx, xlsx, pptx, epub share this)
+    { bytes: [0x50, 0x4b, 0x05, 0x06], type: 'zip' },
+    { bytes: [0xd0, 0xcf, 0x11, 0xe0], type: 'doc' }, // OLE2 (doc, xls, ppt legacy)
+    { bytes: [0x7b, 0x5c, 0x72, 0x74, 0x66], type: 'rtf' }, // {\rtf
+];
+function detectByMagic(buf) {
+    for (const sig of MAGIC) {
+        const match = sig.bytes.every((b, i) => buf[i] === b);
+        if (match)
+            return sig.type;
+    }
+    return null;
+}
+/** Refine a generic 'zip' magic hit using the internal paths */
+async function refineZipType(filePath) {
+    try {
+        const AdmZip = (await Promise.resolve().then(() => __importStar(require('adm-zip')))).default;
+        const zip = new AdmZip(filePath);
+        const names = zip.getEntries().map((e) => e.entryName);
+        if (names.some((n) => n.startsWith('word/')))
+            return 'docx';
+        if (names.some((n) => n.startsWith('xl/')))
+            return 'xlsx';
+        if (names.some((n) => n.startsWith('ppt/')))
+            return 'pptx';
+        if (names.some((n) => n === 'mimetype' || n.startsWith('OEBPS/')))
+            return 'epub';
+        return 'zip';
+    }
+    catch {
+        return 'zip';
+    }
+}
+async function detectDocType(filePath) {
+    const ext = path.extname(filePath).replace('.', '').toLowerCase();
+    if (ext && EXT_MAP[ext] && EXT_MAP[ext] !== 'zip')
+        return EXT_MAP[ext];
+    // Read first 8 bytes for magic sniffing
+    const fd = fs.openSync(filePath, 'r');
+    const buf = Buffer.alloc(8);
+    fs.readSync(fd, buf, 0, 8, 0);
+    fs.closeSync(fd);
+    const magic = detectByMagic(buf);
+    if (magic === 'zip')
+        return refineZipType(filePath);
+    if (magic)
+        return magic;
+    if (ext && EXT_MAP[ext])
+        return EXT_MAP[ext];
+    return 'unknown';
+}
+// ─── Encoding ─────────────────────────────────────────────────────────────────
+async function readTextFile(filePath, hint) {
+    const raw = fs.readFileSync(filePath);
+    if (hint) {
+        return { text: iconv_lite_1.default.decode(raw, hint), encoding: hint };
+    }
+    const detected = chardet_1.default.detect(raw);
+    const enc = detected ?? 'utf-8';
+    try {
+        const text = iconv_lite_1.default.decode(raw, enc);
+        return { text, encoding: enc };
+    }
+    catch {
+        return { text: raw.toString('utf-8'), encoding: 'utf-8 (fallback)' };
+    }
+}
+// ─── Streaming large text ─────────────────────────────────────────────────────
+async function* streamTextFile(filePath, chunkSize = 1024 * 1024) {
+    const stream = fs.createReadStream(filePath, {
+        encoding: 'utf-8',
+        highWaterMark: chunkSize,
+    });
+    for await (const chunk of stream) {
+        yield chunk;
+    }
+}
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+function truncate(text, maxChars) {
+    if (!maxChars || text.length <= maxChars)
+        return { text, truncated: false };
+    return {
+        text: text.slice(0, maxChars) + '\n\n[… content truncated …]',
+        truncated: true,
+    };
+}
+function fileSizeBytes(filePath) {
+    return fs.statSync(filePath).size;
+}
+//# sourceMappingURL=helpers.js.map

package/dist/utils/helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.js","sourceRoot":"","sources":["../../src/utils/helpers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiEA,sCAgBC;AAID,oCAmBC;AAID,wCAWC;AAID,4BASC;AAED,sCAEC;AAxID,uCAAyB;AACzB,2CAA6B;AAC7B,sDAA8B;AAC9B,4DAA+B;AAG/B,kFAAkF;AAElF,MAAM,OAAO,GAA4B;IACvC,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,OAAO;IACd,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,IAAI;IACR,QAAQ,EAAE,IAAI;IACd,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,gEAAgE;AAChE,MAAM,KAAK,GAA8C;IACvD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAY,OAAO;IACnE,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAY,8CAA8C;IAC1G,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;IAChD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAY,8BAA8B;IAC1F,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAM,QAAQ;CACrE,CAAC;AAEF,SAAS,aAAa,CAAC,GAAW;IAChC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,IAAI,KAAK;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,gEAAgE;AAChE,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,wDAAa,SAAS,GAAC,CAAC,CAAC,OAAO,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC9E,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC;QACpE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC;QAClE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC;QACnE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC;QACzF,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClE,IAAI,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK;QAAE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IAEvE,wCAAwC;IACxC,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5B,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAEjB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAExB,IAAI,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iFAAiF;AAE1E,KAAK,UAAU,YAAY,CAChC,QAAgB,EAChB,IAAqB;IAErB,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEtC,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,EAAE,IAAI,EAAE,oBAAK,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC3D,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,oBAAK,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IACvE,CAAC;AACH,CAAC;AAED,iFAAiF;AAE1E,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,QAAgB,EAChB,SAAS,GAAG,IAAI,GAAG,IAAI;IAEvB,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE;QAC3C,QAAQ,EAAE,OAAO;QACjB,aAAa,EAAE,SAAS;KACzB,CAAC,CAAC;IACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAe,CAAC;IACxB,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF,SAAgB,QAAQ,CACtB,IAAY,EACZ,QAAiB;IAEjB,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC5E,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,6BAA6B;QAC7D,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa,CAAC,QAAgB;IAC5C,OAAO,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;AACpC,CAAC"}

package/package.json

@@ -1,6 +1,76 @@
 {
   "name": "doc-reader-sdk",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.0",
+  "description": "Universal document text extractor SDK for Node.js — PDF, DOCX, XLSX, PPTX, CSV, JSON, and more",
+  "type": "commonjs",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "bin": {
+    "doc-reader": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "prebuild": "npm run clean",
+    "dev": "tsx src/cli.ts",
+    "read": "tsx src/cli.ts",
+    "start": "node dist/cli.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:smoke": "tsx src/test.ts",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run typecheck && npm run test && npm run build"
+  },
+  "keywords": [
+    "pdf",
+    "docx",
+    "xlsx",
+    "pptx",
+    "csv",
+    "document",
+    "extractor",
+    "parser",
+    "text-extraction",
+    "sdk"
+  ],
+  "license": "MIT",
+  "author": "sashi-does (https://github.com/sashi-does)",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sashi-does/doc-reader-sdk.git"
+  },
+  "homepage": "https://github.com/sashi-does/doc-reader-sdk#readme",
+  "bugs": {
+    "url": "https://github.com/sashi-does/doc-reader-sdk/issues"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "devDependencies": {
+    "@types/node": "^22.15.3",
+    "@types/pdf-parse": "^1.1.5",
+    "tsx": "^4.23.15",
+    "typescript": "^5.8.3",
+    "vitest": "^5.0.5"
+  },
+  "dependencies": {
+    "adm-zip": "^0.6.1",
+    "chardet": "^2.0.0",
+    "iconv-lite": "^0.6.3",
+    "mammoth": "^1.13.0",
+    "pdf-parse": "^1.1.1",
+    "xlsx": "^0.18.5"
+  }
+}