@dipanshuhandoo/excelio 1.0.0 → 1.1.0

package/dist/esm/index.js

@@ -1,6 +1,8 @@
 // src/writer.js
-import { Readable, PassThrough } from "node:stream";
+import { PassThrough } from "node:stream";
 import { createWriteStream } from "node:fs";
+import { rm } from "node:fs/promises";
+import { setImmediate as tick, setTimeout as sleep } from "node:timers/promises";
 import ExcelJS from "exceljs";
 
 // src/errors.js
@@ -101,11 +103,44 @@ function formatDate(d, fmt) {
   }
 }
 
+// src/rowSource.js
+function rowSourceKind(data) {
+  if (data === void 0 || data === null) return "empty";
+  if (Array.isArray(data)) return "array";
+  if (typeof data !== "object" && typeof data !== "function") return "invalid";
+  if (typeof data[Symbol.asyncIterator] === "function") return "async";
+  if (typeof data[Symbol.iterator] === "function") return "sync";
+  return "invalid";
+}
+var isIterableSource = (data) => {
+  const kind = rowSourceKind(data);
+  return kind === "sync" || kind === "async";
+};
+var specHasIterable = (spec) => Array.isArray(spec) && spec.some((wb) => (Array.isArray(wb?.sheets) ? wb.sheets : []).some((s) => isIterableSource(s?.data)));
+async function closeIterator(iterator) {
+  if (typeof iterator?.return !== "function") return;
+  try {
+    await iterator.return();
+  } catch {
+  }
+}
+function sourceFailure(err, details, signal) {
+  if (signal?.aborted) return new ExcelIoError("Aborted", { code: "ABORTED", cause: err, details });
+  return new ExcelIoError(`Row source failed for sheet "${details.sheetName}": ${err?.message ?? err}`, {
+    code: "SOURCE_ERROR",
+    cause: err,
+    details
+  });
+}
+
 // src/writer.js
 var DEFAULT_BATCH = 5e3;
 var SHEET_NAME_MAX = 31;
+var EXCEL_MAX_DATA_ROWS = 1048575;
+var OUTPUT_HIGH_WATER_MARK = 8 * 1024 * 1024;
+var BACKLOG_POLL_MS = 5;
 async function writeSpec(spec, options = {}) {
-  validateSpec(spec);
+  preflightWrite(spec, options);
   const { to } = options;
   if (to !== void 0 && spec.length > 1) {
     throw new ExcelIoError(
@@ -125,41 +160,106 @@ async function writeSpec(spec, options = {}) {
   return buffers.length === 1 ? buffers[0] : buffers;
 }
 function writeSpecStream(spec, options = {}) {
-  validateSpec(spec);
+  preflightWrite(spec, options);
   const passthrough = new PassThrough();
   writeOneWorkbook(spec[0], 0, passthrough, options).catch((err) => passthrough.destroy(err));
   return passthrough;
 }
+function preflightWrite(spec, options = {}) {
+  validateSpec(spec);
+  validateWriteOptions(options);
+}
 async function writeOneWorkbookToBuffer(workbookSpec, wbIndex, options) {
   const chunks = [];
   const collector = new PassThrough();
   collector.on("data", (chunk) => chunks.push(chunk));
-  await writeOneWorkbook(workbookSpec, wbIndex, collector, options);
+  await writeOneWorkbook(workbookSpec, wbIndex, collector, options, { owned: true });
   return Buffer.concat(chunks);
 }
-async function writeOneWorkbook(workbookSpec, wbIndex, target, options) {
+async function writeOneWorkbook(workbookSpec, wbIndex, target, options, sinkOptions = {}) {
   const { signal, onProgress, beforeWrite, batchSize = DEFAULT_BATCH, logger } = options;
-  const sink = resolveSink(target);
-  const wb = new ExcelJS.stream.xlsx.WorkbookWriter({
-    stream: sink.writable,
-    useStyles: true,
-    useSharedStrings: true
-  });
-  applyWorkbookProperties(wb, workbookSpec.properties);
-  const usedNames = /* @__PURE__ */ new Set();
+  const maxRowsPerSheet = options.maxRowsPerSheet ?? EXCEL_MAX_DATA_ROWS;
   const sheets = workbookSpec.sheets || [];
-  let totalRows = sheets.reduce((n, s) => n + (s.data?.length || 0), 0);
-  let rowsProcessed = 0;
-  for (let sIndex = 0; sIndex < sheets.length; sIndex++) {
-    if (signal?.aborted) throw new ExcelIoError("Aborted", { code: "ABORTED" });
-    const sheetSpec = sheets[sIndex];
-    const sheetName = sanitiseSheetName(sheetSpec.sheet || `Sheet ${sIndex + 1}`, usedNames, logger);
-    const wsOptions = {};
-    if (sheetSpec.styling?.freezeHeader) {
-      wsOptions.views = [{ state: "frozen", ySplit: 1 }];
+  const sink = resolveSink(target, sinkOptions);
+  let wb = null;
+  try {
+    wb = new ExcelJS.stream.xlsx.WorkbookWriter({
+      stream: sink.writable,
+      useStyles: true,
+      useSharedStrings: resolveSharedStrings(options.sharedStrings, sheets)
+    });
+    applyWorkbookProperties(wb, workbookSpec.properties);
+    const used = /* @__PURE__ */ new Set();
+    const firstNames = sheets.map((s, i) => sanitiseSheetName(s.sheet || `Sheet ${i + 1}`, used, logger));
+    const totalRows = computeTotalRows(sheets);
+    const state = { rowsProcessed: 0, last: { sheetIndex: 0, sheetName: firstNames[0], sheetPart: 1 } };
+    const emit = () => onProgress?.({
+      phase: "write",
+      workbookIndex: wbIndex,
+      ...state.last,
+      rowsProcessed: state.rowsProcessed,
+      totalRows
+    });
+    for (let sIndex = 0; sIndex < sheets.length; sIndex++) {
+      throwIfAborted(signal);
+      const sheetSpec = sheets[sIndex];
+      const writer = createSheetWriter({
+        wb,
+        workbookSpec,
+        wbIndex,
+        sIndex,
+        sheetSpec,
+        firstName: firstNames[sIndex],
+        used,
+        beforeWrite,
+        maxRowsPerSheet,
+        onOpen: (sheetName, sheetPart) => {
+          state.last = { sheetIndex: sIndex, sheetName, sheetPart };
+        },
+        onRow: () => {
+          state.rowsProcessed++;
+          if (onProgress && state.rowsProcessed % batchSize === 0) emit();
+        }
+      });
+      const kind = rowSourceKind(sheetSpec.data);
+      if (kind === "sync" || kind === "async") {
+        await writeIterableSheet({ writer, wb, sink, sheetSpec, kind, signal, batchSize, wbIndex, sIndex });
+      } else {
+        const data = sheetSpec.data || [];
+        writer.start(resolveColumns(sheetSpec.columns, data));
+        for (let i = 0; i < data.length; i++) {
+          throwIfAborted(signal);
+          writer.write(data[i]);
+        }
+      }
+      writer.finish();
     }
-    const ws = wb.addWorksheet(sheetName, wsOptions);
-    const cols = resolveColumns(sheetSpec.columns, sheetSpec.data);
+    const committing = wb.commit();
+    committing.catch(() => {
+    });
+    await Promise.race([committing, sink.failed]);
+    if (sink.cleanup) await Promise.race([sink.cleanup(), sink.failed]);
+    sink.release();
+    emit();
+  } catch (err) {
+    await abortWorkbook(wb, sink, err);
+    throw err;
+  }
+}
+function createSheetWriter({ wb, workbookSpec, wbIndex, sIndex, sheetSpec, firstName, used, beforeWrite, maxRowsPerSheet, onOpen, onRow }) {
+  const baseName = sheetSpec.sheet || `Sheet ${sIndex + 1}`;
+  let cols = [];
+  let ws = null;
+  let sheetName = null;
+  let part = 0;
+  let rowsInPart = 0;
+  const open = () => {
+    part += 1;
+    rowsInPart = 0;
+    sheetName = part === 1 ? firstName : sanitiseSheetName(baseName, used, null);
+    const wsOptions = {};
+    if (sheetSpec.styling?.freezeHeader) wsOptions.views = [{ state: "frozen", ySplit: 1 }];
+    ws = wb.addWorksheet(sheetName, wsOptions);
     if (cols.length) {
       ws.columns = cols.map((c) => ({
         key: c.key,
@@ -169,52 +269,188 @@ async function writeOneWorkbook(workbookSpec, wbIndex, target, options) {
       }));
     }
     applyStylingPostColumns(ws, sheetSpec.styling, cols.length);
-    const data = sheetSpec.data || [];
-    for (let i = 0; i < data.length; i++) {
-      if (signal?.aborted) throw new ExcelIoError("Aborted", { code: "ABORTED" });
-      let row = data[i];
+    onOpen(sheetName, part);
+  };
+  return {
+    get sheetName() {
+      return sheetName ?? firstName;
+    },
+    start(resolvedColumns) {
+      cols = resolvedColumns;
+      open();
+    },
+    write(row) {
+      if (rowsInPart >= maxRowsPerSheet) {
+        ws.commit();
+        open();
+      }
+      let out = row;
       if (beforeWrite) {
-        row = beforeWrite(row, {
+        out = beforeWrite(row, {
           workbookIndex: wbIndex,
           workbookName: workbookSpec.workbook,
           sheetIndex: sIndex,
           sheetName,
-          rowNumber: i + 2
+          sheetPart: part,
+          rowNumber: rowsInPart + 2
           // header is row 1
         }) ?? row;
       }
-      const rowValues = cols.length ? cols.reduce((acc, c) => {
-        acc[c.key] = valueForWrite(row[c.key], c.type);
+      const values = cols.length ? cols.reduce((acc, c) => {
+        acc[c.key] = valueForWrite(out[c.key], c.type);
         return acc;
-      }, {}) : row;
-      ws.addRow(rowValues).commit();
-      rowsProcessed++;
-      if (onProgress && rowsProcessed % batchSize === 0) {
-        onProgress({ phase: "write", workbookIndex: wbIndex, sheetIndex: sIndex, rowsProcessed, totalRows });
-      }
+      }, {}) : out;
+      ws.addRow(values).commit();
+      rowsInPart++;
+      onRow();
+    },
+    finish() {
+      ws?.commit();
     }
-    ws.commit();
+  };
+}
+async function writeIterableSheet({ writer, wb, sink, sheetSpec, kind, signal, batchSize, wbIndex, sIndex }) {
+  let rowsRead = 0;
+  const details = () => ({ workbookIndex: wbIndex, sheetIndex: sIndex, sheetName: writer.sheetName, rowsRead });
+  let iterator;
+  try {
+    iterator = kind === "async" ? sheetSpec.data[Symbol.asyncIterator]() : sheetSpec.data[Symbol.iterator]();
+  } catch (err) {
+    throw sourceFailure(err, details(), signal);
   }
-  await wb.commit();
-  if (sink.cleanup) await sink.cleanup();
-  if (onProgress) {
-    onProgress({ phase: "write", workbookIndex: wbIndex, sheetIndex: sheets.length - 1, rowsProcessed, totalRows });
+  const next = async () => {
+    let step;
+    try {
+      step = await iterator.next();
+    } catch (err) {
+      throw sourceFailure(err, details(), signal);
+    }
+    if (step === null || typeof step !== "object") {
+      throw sourceFailure(new TypeError("Iterator result is not an object"), details(), signal);
+    }
+    return step;
+  };
+  let finished = false;
+  try {
+    let step = await next();
+    writer.start(resolveColumns(sheetSpec.columns, step.done ? [] : [step.value]));
+    while (!step.done) {
+      throwIfAborted(signal);
+      writer.write(step.value);
+      rowsRead++;
+      if (rowsRead % batchSize === 0) await waitForCapacity({ wb, sink, signal });
+      step = await next();
+    }
+    finished = true;
+  } finally {
+    if (!finished) await closeIterator(iterator);
   }
 }
-function resolveSink(target) {
+var len = (n) => Number.isFinite(n) ? n : 0;
+var streamLen = (s) => len(s?.writableLength ?? s?._writableState?.length) + len(s?.readableLength ?? s?._readableState?.length);
+function backlogBytes(wb, sink) {
+  let total = streamLen(wb?.zip) + streamLen(wb?.zip?._module) + streamLen(sink?.writable);
+  for (const ws of wb?._worksheets ?? []) {
+    const sb = ws?._stream;
+    if (!sb) continue;
+    for (const buf of sb.buffers ?? []) total += len(buf?.length);
+    for (const p of sb.pipes ?? []) total += streamLen(p);
+  }
+  return total;
+}
+var backlogStats = { peak: 0 };
+async function waitForCapacity({ wb, sink, signal }) {
+  const check = () => {
+    throwIfAborted(signal);
+    if (sink.failure) throw sink.failure;
+  };
+  for (; ; ) {
+    check();
+    const backlog = backlogBytes(wb, sink);
+    if (backlog > backlogStats.peak) backlogStats.peak = backlog;
+    if (backlog <= OUTPUT_HIGH_WATER_MARK) break;
+    await sleep(BACKLOG_POLL_MS);
+  }
+  await tick();
+  check();
+}
+function resolveSink(target, { owned = false } = {}) {
+  const track = (writable, isOwned) => {
+    const sink = { writable, owned: isOwned, failure: null };
+    let onError;
+    sink.failed = new Promise((_, reject) => {
+      onError = (err) => {
+        if (sink.failure) return;
+        sink.failure = err;
+        reject(err);
+      };
+      writable.on("error", onError);
+    });
+    sink.failed.catch(() => {
+    });
+    sink.release = () => {
+      if (!isOwned) writable.removeListener("error", onError);
+    };
+    return sink;
+  };
   if (typeof target === "string") {
     const fileStream = createWriteStream(target);
-    const closed = new Promise((resolve, reject) => {
-      fileStream.once("finish", resolve);
-      fileStream.once("error", reject);
-    });
-    return { writable: fileStream, cleanup: () => closed };
+    const sink = track(fileStream, true);
+    const finished = new Promise((resolve) => fileStream.once("finish", resolve));
+    const closed = new Promise((resolve) => fileStream.once("close", resolve));
+    sink.cleanup = () => finished;
+    sink.remove = async () => {
+      fileStream.destroy();
+      await closed;
+      await rm(target, { force: true });
+    };
+    return sink;
   }
   if (target && typeof target.write === "function") {
-    return { writable: target, cleanup: null };
+    return track(target, owned);
   }
   throw new ExcelIoError(`Invalid write target \u2014 expected file path or writable stream`, { code: "INVALID_SPEC" });
 }
+async function abortWorkbook(wb, sink, err) {
+  const quiet = (s) => s?.on?.("error", () => {
+  });
+  try {
+    if (wb) {
+      wb.zip?.unpipe?.(sink.writable);
+      quiet(wb.zip);
+      quiet(wb.zip?._module);
+      for (const ws of wb._worksheets ?? []) (ws?._stream?.pipes ?? []).forEach(quiet);
+      wb.zip?.abort?.();
+    }
+    if (sink.remove) await sink.remove();
+    else if (sink.owned) sink.writable.destroy?.();
+    sink.release();
+  } catch (cleanupError) {
+    try {
+      if (typeof err === "object" && err !== null || typeof err === "function") err.cleanupError = cleanupError;
+    } catch {
+    }
+  }
+}
+function resolveSharedStrings(option = "auto", sheets) {
+  if (typeof option === "boolean") return option;
+  return !sheets.some((s) => isIterableSource(s?.data));
+}
+function computeTotalRows(sheets) {
+  let total = 0;
+  for (const s of sheets) {
+    const kind = rowSourceKind(s.data);
+    if (kind === "array") total += s.data.length;
+    else if (kind === "sync" || kind === "async") {
+      if (s.totalRows === void 0) return void 0;
+      total += s.totalRows;
+    }
+  }
+  return total;
+}
+function throwIfAborted(signal) {
+  if (signal?.aborted) throw new ExcelIoError("Aborted", { code: "ABORTED" });
+}
 function applyWorkbookProperties(wb, props) {
   if (!props) return;
   if (props.author) wb.creator = props.author;
@@ -258,13 +494,47 @@ function validateSpec(spec) {
   if (!Array.isArray(spec) || spec.length === 0) {
     throw new ExcelIoError("Spec must be a non-empty array of workbooks", { code: "INVALID_SPEC" });
   }
-  for (const wb of spec) {
+  const owners = /* @__PURE__ */ new Map();
+  spec.forEach((wb, wbIndex) => {
     if (!wb || typeof wb !== "object") {
       throw new ExcelIoError("Workbook entries must be objects", { code: "INVALID_SPEC" });
     }
     if (!Array.isArray(wb.sheets) || wb.sheets.length === 0) {
       throw new ExcelIoError("Workbook must have at least one sheet", { code: "INVALID_SPEC" });
     }
+    wb.sheets.forEach((s, i) => {
+      if (!s || typeof s !== "object") {
+        throw new ExcelIoError(`Workbook ${wbIndex}: sheet entries must be objects`, { code: "INVALID_SPEC" });
+      }
+      const name = s.sheet || `Sheet ${i + 1}`;
+      const kind = rowSourceKind(s.data);
+      if (kind === "invalid") {
+        throw new ExcelIoError(`Sheet "${name}": data must be an array or an iterable of row objects`, { code: "INVALID_SPEC" });
+      }
+      if (kind === "sync" || kind === "async") {
+        if (owners.has(s.data)) {
+          throw new ExcelIoError(
+            `Sheet "${name}": its data source is already used by sheet "${owners.get(s.data)}". Pass a separate iterable (or an array) per sheet.`,
+            { code: "INVALID_SPEC" }
+          );
+        }
+        owners.set(s.data, name);
+      }
+      if (s.totalRows !== void 0 && !(Number.isInteger(s.totalRows) && s.totalRows >= 0)) {
+        throw new ExcelIoError(`Sheet "${name}": totalRows must be a non-negative integer`, { code: "INVALID_SPEC" });
+      }
+    });
+  });
+}
+function validateWriteOptions({ maxRowsPerSheet, sharedStrings, batchSize } = {}) {
+  if (maxRowsPerSheet !== void 0 && !(Number.isInteger(maxRowsPerSheet) && maxRowsPerSheet >= 1 && maxRowsPerSheet <= EXCEL_MAX_DATA_ROWS)) {
+    throw new ExcelIoError(`maxRowsPerSheet must be an integer between 1 and ${EXCEL_MAX_DATA_ROWS}`, { code: "INVALID_SPEC" });
+  }
+  if (sharedStrings !== void 0 && sharedStrings !== "auto" && typeof sharedStrings !== "boolean") {
+    throw new ExcelIoError(`sharedStrings must be 'auto', true or false`, { code: "INVALID_SPEC" });
+  }
+  if (batchSize !== void 0 && !(Number.isInteger(batchSize) && batchSize >= 1)) {
+    throw new ExcelIoError("batchSize must be a positive integer", { code: "INVALID_SPEC" });
   }
 }
 
@@ -418,6 +688,10 @@ function decideMode(op, payload, options) {
   if (op === "read" && payload && typeof payload.pipe === "function") {
     return { mode: "inline", reason: "stream input" };
   }
+  if (op === "write" && specHasIterable(payload)) {
+    if (useWorker === true) options.logger?.warn?.("excelio: iterable sheet data is written inline; useWorker ignored");
+    return { mode: "inline", reason: "iterable sheet data" };
+  }
   if (hasHook(options)) return { mode: "inline", reason: "hooks present" };
   if (useWorker === false) return { mode: "inline", reason: "disabled" };
   if (useWorker === true) return { mode: "worker", reason: "forced" };
@@ -445,7 +719,7 @@ function estimateRows(spec) {
   let n = 0;
   for (const wb of spec) {
     for (const sheet of wb?.sheets || []) {
-      n += sheet.data?.length || 0;
+      n += Array.isArray(sheet?.data) ? sheet.data.length : 0;
     }
   }
   return n;
@@ -521,13 +795,16 @@ var excelio = {
    * is provided and returns the target.
    */
   async write(spec, options = {}) {
+    preflightWrite(spec, options);
     return maybeUseWorker("write", spec, options, () => writeSpec(spec, options));
   },
   /**
    * Returns a Readable stream for the first workbook. Synchronous return;
-   * errors during the underlying write surface via the stream's 'error' event.
+   * spec errors throw immediately, errors during the underlying write surface
+   * via the stream's 'error' event.
    */
   writeStream(spec, options = {}) {
+    preflightWrite(spec, options);
     return writeSpecStream(spec, options);
   },
   /**