@reportwright/engine 0.0.0-stage → 0.12.0

package/examples/stream-1m.mjs

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+// A million-row ledger streamed to a PDF file, with the process's peak memory: the proof that a large report's PDF
+// is written with flat memory. The rows are made as they are read (an async generator standing in for a database
+// cursor), so nothing holds them all.
+//
+//   npm i @reportwright/engine @reportwright/fonts
+//   node node_modules/@reportwright/engine/examples/stream-1m.mjs [rows=1000000] [out=ledger.pdf] [--plain] [--grouped] [--tagged] [--parallel=4]
+//
+// Prints one JSON line: { rows, pages, streamed, peakRssMB, ms, bytes, ... }. "Page N of M" is in the page footer
+// (the rows are read twice: once to count the pages); --plain leaves M out (one pass). --grouped adds two group levels
+// with subtotals; --tagged makes an accessible PDF/A (PDF/UA-1 + PDF/A-2a); --parallel=N lays out and paints with N
+// worker threads. RW_ENGINE: import the engine from another path (a checkout: packages/engine/entry.js); RW_PUBLIC: a
+// folder with fonts/ and harfbuzz-subset.wasm instead of @reportwright/fonts.
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const args = process.argv.slice(2);
+const flag = (f) => args.includes(`--${f}`);
+const opt = (f) => args.find(a => a.startsWith(`--${f}=`))?.split('=')[1];
+const pos = args.filter(a => !a.startsWith('--'));
+const N = Number(pos[0]) || 1_000_000;
+const file = pos[1] || 'ledger.pdf';
+const engine = await import(process.env.RW_ENGINE ? pathToFileURL(path.resolve(process.env.RW_ENGINE)).href : '@reportwright/engine');
+const { exportPdfStream, FontStore, fontFile, createSubsetter } = engine;
+
+// fonts: @reportwright/fonts (the engine finds it), or a folder of them
+let fonts, subset, icc;
+if (process.env.RW_PUBLIC) {
+  const dir = path.resolve(process.env.RW_PUBLIC);
+  fonts = new FontStore(async (k) => fs.promises.readFile(path.join(dir, 'fonts', fontFile(k))));
+  subset = createSubsetter(() => fs.promises.readFile(path.join(dir, 'harfbuzz-subset.wasm')));
+  icc = fs.readFileSync(path.join(dir, 'icc', 'sRGB-v2-magic.icc'));
+}
+
+const REGIONS = ['North', 'South', 'East', 'West'], TYPES = ['Deposit', 'Withdrawal', 'Transfer', 'Fee', 'Interest'];
+/** the rows, a thousand at a time, made as they are asked for */
+async function* rows() {
+  for (let i = 0; i < N; i += 1000) {
+    const batch = [];
+    for (let j = i; j < Math.min(N, i + 1000); j++) {
+      batch.push({ id: j + 1, account: `AC${String(10000 + (j * 7) % 9000)}`, region: REGIONS[(j >> 9) % 4], type: TYPES[(j >> 6) % 5],
+        date: new Date(Date.UTC(2025, j % 12, 1 + (j % 28))), amount: Math.round(((j * 7919) % 200000 - 100000) * 1.37) / 100,
+        reference: j % 37 === 0 ? `Reversal of an earlier transfer between linked accounts, batch ${j}` : `REF-${j * 31}` });
+    }
+    yield batch;
+  }
+}
+const grouped = flag('grouped');
+const total = (label) => [{ height: 15, style: { fontWeight: 'bold' }, cells: [{ value: label }, { value: '=CountRows()' }, null, null, null, { value: '=Sum(Fields.amount)', style: { format: 'N2', textAlign: 'right' } }, null] }];
+const def = {
+  $schema: 'pagewright/report@1', name: 'Ledger', locale: 'en-IN', page: { size: 'A4', margins: [36, 36, 36, 36] },
+  // the rows come from opts.sqlRows below (a cursor); the query text is not run
+  dataSources: [{ name: 'db', type: 'sql', connection: 'ledger', query: 'select * from transactions order by id' }],
+  dataSets: [{ name: 'Tx', source: 'db', fields: [{ name: 'id', type: 'number' }, { name: 'account', type: 'string' }, { name: 'region', type: 'string' }, { name: 'type', type: 'string' },
+    { name: 'date', type: 'date' }, { name: 'amount', type: 'number' }, { name: 'reference', type: 'string' }] }],
+  sections: {
+    pageHeader: { height: 22, items: [{ type: 'textbox', name: 'Title', x: 0, y: 0, w: 300, h: 18, value: 'Transactions ledger', style: { fontWeight: 'bold', fontSize: 12 } }] },
+    body: { items: [{ type: 'table', name: 'Tx', dataSet: 'Tx', x: 0, y: 0, w: 523, h: 40,
+      columns: [{ width: 50 }, { width: 62 }, { width: 50 }, { width: 62 }, { width: 62 }, { width: 70 }, { width: 167 }],
+      header: [{ height: 16, style: { fontWeight: 'bold', backgroundColor: '#eef2f7' }, cells: ['Id', 'Account', 'Region', 'Type', 'Date', 'Amount', 'Reference'].map(value => ({ value })) }],
+      detail: [{ height: 14, style: { borderBottom: '0.5 solid #e5e7eb' }, cells: [{ value: '=Fields.id' }, { value: '=Fields.account' }, { value: '=Fields.region' }, { value: '=Fields.type' },
+        { value: '=Fields.date', style: { format: 'd' } }, { value: '=Fields.amount', style: { format: 'N2', textAlign: 'right' } }, { value: '=Fields.reference', style: { fontSize: 8 } }] }],
+      footer: total('Total'),
+      ...(grouped ? { groups: [
+        { name: 'Region', expr: '=Fields.region', repeatHeader: true, header: [{ height: 18, style: { fontWeight: 'bold', backgroundColor: '#dbeafe' }, cells: [{ value: '="Region " & Fields.region', colSpan: 3 }, null, null] }], footer: total('Region total') },
+        { name: 'Type', expr: '=Fields.type', header: [{ height: 15, cells: [null, { value: '=Fields.type' }] }], footer: total('Type total') },
+      ] } : {}) }] },
+    pageFooter: { height: 18, items: [{ type: 'textbox', name: 'Page', x: 0, y: 2, w: 523, h: 14, style: { textAlign: 'right' },
+      value: flag('plain') ? 'Page {Globals.PageNumber}' : 'Page {Globals.PageNumber} of {Globals.TotalPages}' }] },
+  },
+};
+
+let peak = 0;
+const rss = () => { const m = process.memoryUsage.rss() / 1048576; if (m > peak) peak = m; };
+rss();
+const base = peak;
+const timer = setInterval(rss, 20);
+const t0 = Date.now();
+const out = fs.createWriteStream(file);
+const r = await exportPdfStream(def, { fonts, subset, title: 'Ledger', sqlRows: () => rows(), maxRows: Math.max(N, 5_000_000), timeoutMs: 24 * 3600e3,
+  ...(flag('tagged') ? { tagged: { lang: 'en-IN' }, pdfa: icc ? { icc } : true } : {}), ...(opt('parallel') ? { parallel: Number(opt('parallel')) } : {}) }, out);
+await new Promise((ok, fail) => out.end((e) => (e ? fail(e) : ok())));
+clearInterval(timer);
+rss();
+console.log(JSON.stringify({ rows: N, pages: r.pages, streamed: r.streamed, ...(r.why ? { why: r.why } : {}), peakRssMB: Math.round(peak), startRssMB: Math.round(base), ms: Date.now() - t0,
+  bytes: fs.statSync(file).size, file, node: process.version, grouped, tagged: flag('tagged'), parallel: Number(opt('parallel')) || 1 }));

package/index.d.ts

@@ -0,0 +1,3 @@
+// @reportwright/engine types: the API (generated from the engine's JSDoc into dist/types) and the report schema.
+export * from './dist/types/packages/engine/entry.js';
+export type * from './report.js';

package/package.json

@@ -1,6 +1,81 @@
 {
   "name": "@reportwright/engine",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.12.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "license": "MIT",
+  "author": "Arun Mallikarjun",
+  "homepage": "https://github.com/MrArun005/reportwright/tree/main/packages/engine#readme",
+  "bugs": {
+    "url": "https://github.com/MrArun005/reportwright/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/MrArun005/reportwright.git",
+    "directory": "packages/engine"
+  },
+  "keywords": [
+    "reportwright",
+    "report",
+    "reporting",
+    "pdf",
+    "pdf-a",
+    "excel",
+    "xlsx",
+    "docx",
+    "pptx",
+    "report-engine",
+    "jasperreports",
+    "ssrs",
+    "rdl"
+  ],
+  "description": "ReportWright report engine: JSON report definitions to pages, PDF (incl. PDF/A), Word, Excel, PowerPoint, HTML and CSV. Node and browsers. Fonts: @reportwright/fonts; CLI: @reportwright/cli.",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./report": {
+      "types": "./report.d.ts"
+    },
+    "./schema.json": "./schema.json",
+    "./pool": {
+      "types": "./pool/index.d.ts",
+      "default": "./pool/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "types": "./index.d.ts",
+  "files": [
+    "dist",
+    "pool",
+    "examples",
+    "index.d.ts",
+    "report.d.ts",
+    "schema.json",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=20.9"
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "pdf-lib": "1.17.1",
+    "@pdf-lib/fontkit": "1.1.1",
+    "bwip-js": "4.11.4",
+    "fflate": "0.8.3",
+    "bidi-js": "1.0.3"
+  },
+  "peerDependencies": {
+    "@reportwright/fonts": "0.12.0"
+  },
+  "peerDependenciesMeta": {
+    "@reportwright/fonts": {
+      "optional": true
+    }
+  }
+}

package/pool/index.d.ts

@@ -0,0 +1,23 @@
+// @reportwright/engine/pool (Node only): render many reports in parallel on worker threads.
+import type { Report } from '../report.js';
+export interface PoolRenderOptions {
+  /** default pdf */
+  format?: 'pdf' | 'xlsx' | 'docx' | 'pptx' | 'csv' | 'html';
+  parameters?: Record<string, unknown>;
+  timeZone?: string;
+  timeoutMs?: number;
+  /** passed to the exporter: exportPdf's title, pdfa, … */
+  options?: Record<string, unknown>;
+}
+export interface Pool {
+  /** the file's bytes */
+  render(definition: Report | object, opts?: PoolRenderOptions): Promise<Uint8Array>;
+  /** stops the workers; renders still waiting are rejected */
+  close(): Promise<void>;
+  readonly size: number;
+  /** renders waiting or running */
+  readonly pending: number;
+}
+/** workers: default the CPU count less one. fontsDir: a folder of the fonts (default @reportwright/fonts).
+ *  heapMB: each worker's heap limit (default 1024). timeoutMs: stop a render past it with status 504 (default 0: no limit). */
+export function createPool(opt?: { workers?: number; fontsDir?: string; maxQueue?: number; heapMB?: number; timeoutMs?: number }): Pool;

package/pool/index.js

@@ -0,0 +1,85 @@
+// @reportwright/engine/pool (Node only): render many reports in parallel on worker threads.
+//   const pool = createPool({ workers: 4 });
+//   const pdf = await pool.render(definition, { format: 'pdf', parameters: { id: 42 } });
+//   await pool.close();
+import { Worker } from 'node:worker_threads';
+import os from 'node:os';
+
+/**
+ * @typedef {{ format?: 'pdf' | 'xlsx' | 'docx' | 'pptx' | 'csv' | 'html', parameters?: Record<string, unknown>,
+ *   timeZone?: string, timeoutMs?: number, options?: Record<string, unknown> }} PoolRenderOptions
+ *   options: passed to the exporter (exportPdf's title, pdfa, …)
+ * @typedef {{ render: (definition: object, opts?: PoolRenderOptions) => Promise<Uint8Array>, close: () => Promise<void>, readonly size: number, readonly pending: number }} Pool
+ */
+
+/**
+ * A pool of worker threads, each with the engine and its fonts loaded once.
+ * @param {{ workers?: number, fontsDir?: string, maxQueue?: number, heapMB?: number, timeoutMs?: number }} [opt]
+ *   workers: default the CPU count less one. fontsDir: a folder of the fonts (default @reportwright/fonts).
+ *   maxQueue: renders waiting beyond it are refused (default 10,000).
+ *   heapMB: each worker's heap limit (default 1024): a render past it fails alone (its worker is replaced), instead of
+ *   taking the whole process down. timeoutMs: a render running longer is stopped (status 504) and its worker replaced
+ *   (default: no limit; the engine's own opts.timeoutMs per render is checked between rows, this one always holds).
+ * @returns {Pool}
+ */
+export function createPool({ workers = Math.max(1, (os.availableParallelism?.() ?? os.cpus().length) - 1), fontsDir, maxQueue = 10000, heapMB = 1024, timeoutMs = 0 } = {}) {
+  if (!Number.isInteger(workers) || workers < 1) throw new TypeError('createPool({ workers }): workers must be a whole number of at least 1');
+  if (!Number.isFinite(heapMB) || heapMB < 32) throw new TypeError('createPool({ heapMB }): heapMB must be a number of at least 32');
+  if (!Number.isFinite(timeoutMs) || timeoutMs < 0) throw new TypeError('createPool({ timeoutMs }): timeoutMs must be 0 (no limit) or more');
+  /** @type {{ def: object, opts: PoolRenderOptions, resolve: (b: Uint8Array) => void, reject: (e: Error) => void }[]} */
+  const queue = [];
+  /** @type {Set<Worker>} */
+  const idle = new Set();
+  /** @type {Map<Worker, (typeof queue)[number]>} */
+  const busy = new Map();
+  /** @type {Map<Worker, any>} */
+  const timers = new Map();
+  let closed = false;
+  const done = (w) => { clearTimeout(timers.get(w)); timers.delete(w); const job = busy.get(w); busy.delete(w); return job; };
+  const spawn = () => {
+    const w = new Worker(new URL('./worker.js', import.meta.url), { workerData: { fontsDir }, resourceLimits: { maxOldGenerationSizeMb: heapMB } });
+    w.on('message', (msg) => {
+      const job = done(w);
+      if (job) msg.error ? job.reject(Object.assign(new Error(msg.error.message), msg.error)) : job.resolve(new Uint8Array(msg.bytes));
+      idle.add(w);
+      next();
+    });
+    w.on('error', (e) => {
+      const oom = /** @type {any} */ (e)?.code === 'ERR_WORKER_OUT_OF_MEMORY';
+      done(w)?.reject(oom ? Object.assign(new Error(`the render ran out of memory (heapMB: ${heapMB}) and was stopped`), { status: 500, code: e.code }) : e);
+    });
+    w.on('exit', () => {
+      idle.delete(w);
+      done(w)?.reject(new Error('the render worker stopped'));
+      if (!closed) { idle.add(spawn()); next(); } // a crashed worker is replaced
+    });
+    return w;
+  };
+  const next = () => {
+    while (queue.length && idle.size) {
+      const w = /** @type {Worker} */ (idle.values().next().value);
+      idle.delete(w);
+      const job = /** @type {(typeof queue)[number]} */ (queue.shift());
+      busy.set(w, job);
+      // past the time limit the worker is stopped (a render in a long synchronous step cannot be interrupted otherwise)
+      if (timeoutMs > 0) timers.set(w, setTimeout(() => { done(w)?.reject(Object.assign(new Error(`the render ran longer than ${timeoutMs} ms and was stopped`), { status: 504 })); w.terminate(); }, timeoutMs));
+      w.postMessage({ def: job.def, opts: job.opts });
+    }
+  };
+  for (let i = 0; i < workers; i++) idle.add(spawn());
+  return {
+    get size() { return workers; },
+    get pending() { return queue.length + busy.size; },
+    render(def, opts = {}) {
+      if (closed) return Promise.reject(new Error('the pool is closed'));
+      if (!def || typeof def !== 'object') return Promise.reject(new TypeError('pool.render(definition, opts): definition must be a report object'));
+      if (queue.length >= maxQueue) return Promise.reject(new Error(`the pool queue is full (${maxQueue} renders waiting)`));
+      return new Promise((resolve, reject) => { queue.push({ def, opts, resolve, reject }); next(); });
+    },
+    async close() {
+      closed = true;
+      for (const job of queue.splice(0)) job.reject(new Error('the pool is closed'));
+      await Promise.all([...idle, ...busy.keys()].map(w => w.terminate()));
+    },
+  };
+}

package/pool/worker.js

@@ -0,0 +1,27 @@
+// One render thread of @reportwright/engine/pool: the engine and the fonts load once, then one job at a time.
+import { parentPort, workerData } from 'node:worker_threads';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { render, FontStore, fontFile, defaultFontStore, exportPdf, exportXlsx, exportDocx, exportPptx, exportHtml, exportCsv, needsExportData } from '@reportwright/engine';
+
+const fonts = workerData?.fontsDir ? new FontStore((key) => fs.readFile(path.join(workerData.fontsDir, fontFile(key)))) : defaultFontStore();
+const EXPORT = {
+  pdf: (m, o) => exportPdf(m, { fonts, ...o }),
+  xlsx: (m, o) => exportXlsx(m, { fonts, ...o }),
+  docx: (m, o) => exportDocx(m, { fonts, ...o }),
+  pptx: (m, o) => exportPptx(m, { fonts, ...o }),
+  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, fonts, o)),
+  csv: async (m, o) => new TextEncoder().encode(exportCsv(m, o)),
+};
+/** @type {import('node:worker_threads').MessagePort} */ (parentPort).on('message', async ({ def, opts }) => {
+  try {
+    const format = opts.format || 'pdf';
+    if (!EXPORT[format]) throw new Error(`unknown format "${format}" (pdf, xlsx, docx, pptx, html, csv)`);
+    const model = await render(def, { fontStore: fonts, parameters: opts.parameters, timeZone: opts.timeZone, timeoutMs: opts.timeoutMs, exportData: needsExportData(format, opts.options) });
+    const bytes = await EXPORT[format](model, opts.options || {});
+    const buf = bytes.buffer.byteLength === bytes.byteLength ? bytes.buffer : bytes.slice().buffer;
+    parentPort?.postMessage({ bytes: buf }, [buf]);
+  } catch (e) {
+    parentPort?.postMessage({ error: { message: String(e?.message || e), status: e?.status } });
+  }
+});