@reportwright/engine 0.11.0 → 0.12.1

package/dist/types/src/exporters/pdfstream.d.ts

@@ -0,0 +1,199 @@
+/**
+ * @typedef {{ write(u8: Uint8Array): any, once?: (ev: string, cb: () => void) => any }} Sink
+ *   write may return a promise (awaited: back-pressure), or false with a 'drain' event to wait for (a Node Writable).
+ * @typedef {{ title: string, page: number, top?: number, level?: number }} OutlineEntry
+ *   page: 1-based; top: the destination's y in PDF space (from the bottom), else the page fits the window.
+ */
+/**
+ * @param {Sink} sink
+ * @param {{ fontStore: import('../engine/text/fonts.js').FontStore, fonts?: string[],
+ *   subset?: (font: Uint8Array, codePoints: Iterable<number>, opt?: { glyphs?: Iterable<number> }) => Promise<Uint8Array>,
+ *   id?: string }} opt
+ *   fonts: font keys to name now (F1, F2… in this order); any other font is named on first use.
+ *   subset: subset.js's subsetter; without it fonts embed in full. id: the file ID (32 hex digits), else random.
+ */
+export function createPdfStream(sink: Sink, opt: {
+    fontStore: import("../engine/text/fonts.js").FontStore;
+    fonts?: string[];
+    subset?: (font: Uint8Array, codePoints: Iterable<number>, opt?: {
+        glyphs?: Iterable<number>;
+    }) => Promise<Uint8Array>;
+    id?: string;
+}): {
+    /** @param {string} key @returns {string} the font's resource name (F1…) */
+    fontName: (key: string) => string;
+    /**
+     * The text's glyphs, one per character (as the engine measured it: no layout features), as hex for <…> Tj, with
+     * the glyphs recorded for the subset, W and ToUnicode.
+     * @param {string} key @param {string} text
+     */
+    encode(key: string, text: string): string;
+    /**
+     * The same glyphs as a literal string's body, two bytes each, escaped (as pdf.js draws plain text): ( … ) Tj.
+     * @param {string} key @param {string} text
+     */
+    literal(key: string, text: string): string;
+    /**
+     * Shaped glyphs drawn by ID (conjuncts, joined forms): [glyph ID, the text it stands for ('' for none)].
+     * @param {string} key @param {Iterable<[number, string]>} glyphs
+     */
+    addGlyphs(key: string, glyphs: Iterable<[number, string]>): void;
+    /**
+     * A name in the shared Resources dictionary for a value (an indirect ref "12 0 R" or an inline dictionary).
+     * @param {'ExtGState'|'XObject'|'Pattern'|'Shading'|'ColorSpace'} kind @param {string} value
+     */
+    resource(kind: "ExtGState" | "XObject" | "Pattern" | "Shading" | "ColorSpace", value: string, name?: any): string;
+    /**
+     * The ExtGState name for fill and stroke opacity, named by its values (the same name in every writer: pages
+     * painted elsewhere, parallel.js, use it too). @param {number} ca @param {number} CA
+     */
+    gs: (ca: number, CA: number) => string;
+    /** What this writer's pages used, for another writer to take over (pages painted elsewhere). */
+    used: () => {
+        fonts: (string | number[] | [number, string][])[][];
+        gs: string[];
+    };
+    /** Take over what pages painted elsewhere used (used() of their writer). @param {any} u */
+    take(u: any): void;
+    /**
+     * Write an object now (an image: its dictionary and its stream, already filtered). @returns {Promise<number>} its number
+     * @param {string} dict @param {Uint8Array} [stream]
+     */
+    object(dict: string, stream?: Uint8Array): Promise<number>;
+    /** An object number to write later (objectAt). */
+    reserve: () => number;
+    /** Write a reserved object. @param {number} n @param {string} dict @param {Uint8Array} [stream] */
+    objectAt: (n: number, dict: string, stream?: Uint8Array) => Promise<void>;
+    /**
+     * Write a reserved object whose text may be long (an array of every row's element): head, the parts as they come,
+     * tail; written out in place, never held whole. @param {number} n @param {string} head @param {Iterable<string>} parts @param {string} tail
+     */
+    bigObject(n: number, head: string, parts: Iterable<string>, tail: string): Promise<void>;
+    /** The object number of page i (0-based), written or not: links and outline entries may point ahead. @param {number} i */
+    pageRef(i: number): number;
+    /**
+     * The total page count, drawn at the end: a Form XObject whose origin is the text's baseline at the box's left;
+     * draw it with `q 1 0 0 1 x y cm /<name> Do Q`. align: within width.
+     * @param {{ font: string, size: number, width: number, align?: 'left'|'center'|'right', color?: string }} o
+     */
+    totalPages(o: {
+        font: string;
+        size: number;
+        width: number;
+        align?: "left" | "center" | "right";
+        color?: string;
+    }): string;
+    /**
+     * Write a page: its content stream (deflated) and its dictionary, now. Nothing of it stays but two offsets.
+     * @param {{ width: number, height: number, content?: string | Uint8Array, deflated?: Uint8Array, annots?: (string | { ref: number, dict: string })[], extra?: string }} p
+     *   deflated: the content stream already compressed (zlib), as pages painted elsewhere come.
+     *   content: the page's operators (a string holds bytes 0–255, as pdf.js's Content makes). annots: annotation
+     *   dictionaries (with a reserved number: { ref, dict }). extra: more entries for the page dictionary (/StructParents 3).
+     */
+    addPage(p: {
+        width: number;
+        height: number;
+        content?: string | Uint8Array;
+        deflated?: Uint8Array;
+        annots?: (string | {
+            ref: number;
+            dict: string;
+        })[];
+        extra?: string;
+    }): Promise<void>;
+    /**
+     * The end of the file: totals, fonts, resources, page tree, outline, catalog, Info, xref, trailer.
+     * @param {{ outline?: OutlineEntry[], info?: { title?: string, author?: string, subject?: string, creationDate?: Date }, catalog?: string }} [o]
+     *   catalog: more entries for the catalog (the structure tree, metadata, output intents).
+     * @returns {Promise<{ pages: number, bytes: number }>}
+     */
+    finish(o?: {
+        outline?: OutlineEntry[];
+        info?: {
+            title?: string;
+            author?: string;
+            subject?: string;
+            creationDate?: Date;
+        };
+        catalog?: string;
+    }): Promise<{
+        pages: number;
+        bytes: number;
+    }>;
+};
+/**
+ * A report to PDF bytes written to sink as its pages are laid out (planPaged: a flowing table, one window of rows at a
+ * time), else rendered whole and exported by exportPdf, the same file it always was, written to sink in one piece.
+ * opts: render()'s options and exportPdf's (fontStore, subset, title, author, creationDate, drillBase…). Tagged
+ * (PDF/UA-1) and PDF/A files stream too; encrypted and signed files, save options and PDF/A without a colour profile
+ * take the whole-model path (the result's why says which).
+ * @param {any} def @param {any} opts @param {Sink} sink
+ * @returns {Promise<{ streamed: boolean, pages: number, warnings: string[], why?: string }>}
+ */
+export function exportPdfStream(def: any, opts: any, sink: Sink): Promise<{
+    streamed: boolean;
+    pages: number;
+    warnings: string[];
+    why?: string;
+}>;
+/**
+ * The pages of a paged plan (planPaged) as a PDF written to sink, page by page.
+ * @param {any} plan @param {any} opts fontStore, subset, title, author, creationDate, drillBase @param {Sink} sink
+ * @returns {Promise<{ pages: number, warnings: string[] }>}
+ */
+export function writePagedPdf(plan: any, opts: any, sink: Sink): Promise<{
+    pages: number;
+    warnings: string[];
+}>;
+/**
+ * A page of the model as its content stream's operators and its link annotations, drawn with the shared painter
+ * (pdfpaint.js) as exportPdf draws it. env (paintEnv): the writer's names and glyph records, the page refs links jump
+ * to, the tagger when tagged. Pages painted in parallel (parallel.js) use a writer of their own and the same names.
+ * @param {any} pg @param {number} H @param {any} env
+ * @returns {{ content: string, annots: any[] }}
+ */
+export function paintPage(pg: any, H: number, env: any): {
+    content: string;
+    annots: any[];
+};
+/**
+ * What paintPage draws with: a writer's resource names and glyph records (w), the fonts, shaped glyphs' texts.
+ * @param {any} w @param {any} fontStore @param {{ drillBase?: string, pageRef?: (i: number) => any, tags?: any }} [o]
+ */
+export function paintEnv(w: any, fontStore: any, o?: {
+    drillBase?: string;
+    pageRef?: (i: number) => any;
+    tags?: any;
+}): {
+    shared: {
+        font: (f: any) => any;
+        gs: (ca: any, CA: any) => any;
+        image: () => never;
+    };
+    pf: (key: string) => any;
+    font: (key: string) => any;
+    shaped: Map<string, Map<number, string>>;
+    shapedOf: (key: string) => Map<number, string>;
+    vmetrics: (key: string) => any;
+    literal: (key: string, text: string) => any;
+    pageRef: (i: number) => any;
+    reserve: () => any;
+    drillBase: string;
+    tags: any;
+};
+/**
+ * write may return a promise (awaited: back-pressure), or false with a 'drain' event to wait for (a Node Writable).
+ */
+export type Sink = {
+    write(u8: Uint8Array): any;
+    once?: (ev: string, cb: () => void) => any;
+};
+/**
+ * page: 1-based; top: the destination's y in PDF space (from the bottom), else the page fits the window.
+ */
+export type OutlineEntry = {
+    title: string;
+    page: number;
+    top?: number;
+    level?: number;
+};

package/dist/types/src/exporters/pdfstreamtags.d.ts

@@ -0,0 +1,45 @@
+/**
+ * The structure tree of a streamed, tagged PDF (PDF/UA-1, and PDF/A-2a with it): pdfua.js's tree for the shapes that
+ * stream (engine/paged.js), written as the pages are. Document > P / H1–H6 (text boxes), Table > TR > TH/TD (the table's
+ * header rows TH with Scope Column the first time, an artifact where they repeat; empty cells as empty TD; spans as
+ * ColSpan), Link > OBJR inside the element whose text it covers; page header and footer, borders and boxes artifacts.
+ * Each page's rows (TR, TD), its parent-tree array and its links' elements are written when the page ends; what stays
+ * is one number per row (the Table's kids) and two per page and link (the parent tree), written at the end.
+ * A cell's element is a few dozen bytes in an object stream (no /Type, which is optional; no /A on a plain TD): a
+ * million-row table has some 8 million of them, the most of the file's objects.
+ */
+/**
+ * @param {any} w the PDF writer (pdfstream.js createPdfStream)
+ * @param {{ maxElements?: number }} [o] maxElements: past it the export stops (413)
+ */
+export function createStreamTagger(w: any, { maxElements }?: {
+    maxElements?: number;
+}): {
+    /** structure elements so far */
+    readonly count: number;
+    /** a new page (window: the paged window it came from; row ids are its own) @param {number} window */
+    page(window: number): void;
+    /**
+     * The marked content that opens item it: a structure element's MCID, or an artifact; null for a link.
+     * @param {any} it
+     */
+    open(it: any): string;
+    /**
+     * A link annotation of the page: its Link element (in the element of the text it covers, else the Document).
+     * @returns {number} the annotation's StructParent
+     * @param {any} it @param {number} annot the annotation's object number
+     */
+    link(it: any, annot: number): number;
+    /**
+     * The page is drawn: its elements and parent-tree array are written. @returns {Promise<string>} the page's entries
+     * @param {number} pageRef
+     */
+    end(pageRef: number): Promise<string>;
+    /**
+     * The tree's top: Table, Document, the parent tree, the root. @returns {Promise<string>} the catalog's entries
+     * @param {{ lang?: string }} o
+     */
+    finish({ lang }?: {
+        lang?: string;
+    }): Promise<string>;
+};

package/dist/types/src/exporters/spill.d.ts

@@ -0,0 +1,20 @@
+/** At the server's start: remove the spill folders of processes that are gone (pw-spill-<pid>-…) and old-style ones. */
+export function sweepAtBoot(root?: any): void;
+export function spillSorter({ run, runBytes, frame, maxMB, totalMB, dir: root, staleMs }?: {
+    run?: number;
+    runBytes?: number;
+    frame?: number;
+    maxMB?: number;
+    totalMB?: number;
+    dir?: any;
+    staleMs?: number;
+}): (cmp: (a: any, b: any) => number) => {
+    /** @param {any} r */
+    push(r: any): Promise<void>;
+    drain(): AsyncGenerator<any, void, unknown>;
+    close(): Promise<void>;
+    /** Tests: the runs written so far. */
+    readonly runs: number;
+};
+export function canSpill(): boolean;
+export function spillShared(): any;

package/dist/types/src/exporters/subset.d.ts

@@ -2,4 +2,5 @@
 export function createSubsetter(loadWasm: () => Promise<ArrayBuffer | Uint8Array>): (font: Uint8Array, codePoints: Iterable<number>, opt?: {
     glyphs?: Iterable<number>;
     layout?: boolean;
+    weight?: number;
 }) => Promise<Uint8Array>;

package/examples/stream-1m.mjs

@@ -0,0 +1,100 @@
+#!/usr/bin/env node
+// A million-row ledger streamed to a PDF file, with the process's peak memory: the proof that a large report's PDF
+// is written with flat memory. The rows are made as they are read (an async generator standing in for a database
+// cursor), so nothing holds them all.
+//
+//   npm i @reportwright/engine @reportwright/fonts
+//   node node_modules/@reportwright/engine/examples/stream-1m.mjs [rows=1000000] [out=ledger.pdf] [--plain] [--grouped] [--presorted] [--tagged] [--array[=text]] [--parallel=N]
+//
+// Prints one JSON line: { rows, pages, streamed, peakRssMB, ms, bytes, ... }. "Page N of M" is in the page footer
+// (the rows are read twice: once to count the pages); --plain leaves M out (one pass). --grouped adds two group levels
+// with subtotals (sorted through a disk spill; --presorted: the rows come in group order and are only checked); --tagged
+// makes an accessible PDF/A (PDF/UA-1 + PDF/A-2a); --array: the rows as one array in the definition (--array=text: as
+// JSON text); --parallel=N lays out and paints with N worker threads (serial by default: each thread costs 150-300 MB).
+// RW_ENGINE: import the engine from another path (a checkout: packages/engine/entry.js); RW_PUBLIC: a folder with fonts/
+// and harfbuzz-subset.wasm instead of @reportwright/fonts.
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const args = process.argv.slice(2);
+const flag = (f) => args.includes(`--${f}`);
+const opt = (f) => args.find(a => a.startsWith(`--${f}=`))?.split('=')[1];
+const pos = args.filter(a => !a.startsWith('--'));
+const N = Number(pos[0]) || 1_000_000;
+const file = pos[1] || 'ledger.pdf';
+const engine = await import(process.env.RW_ENGINE ? pathToFileURL(path.resolve(process.env.RW_ENGINE)).href : '@reportwright/engine');
+const { exportPdfStream, FontStore, fontFile, createSubsetter } = engine;
+
+// fonts: @reportwright/fonts (the engine finds it), or a folder of them
+let fonts, subset, icc;
+if (process.env.RW_PUBLIC) {
+  const dir = path.resolve(process.env.RW_PUBLIC);
+  fonts = new FontStore(async (k) => fs.promises.readFile(path.join(dir, 'fonts', fontFile(k))));
+  subset = createSubsetter(() => fs.promises.readFile(path.join(dir, 'harfbuzz-subset.wasm')));
+  icc = fs.readFileSync(path.join(dir, 'icc', 'sRGB-v2-magic.icc'));
+}
+
+const presorted = flag('presorted');
+const REGIONS = ['North', 'South', 'East', 'West'], TYPES = ['Deposit', 'Withdrawal', 'Transfer', 'Fee', 'Interest'];
+/** the rows, a thousand at a time, made as they are asked for */
+async function* rows() {
+  for (let i = 0; i < N; i += 1000) {
+    const batch = [];
+    for (let j = i; j < Math.min(N, i + 1000); j++) {
+      // --presorted: the rows come in group order (as a query ordered by region, type gives them)
+      const region = presorted ? REGIONS[Math.floor(j * 4 / N)] : REGIONS[(j >> 9) % 4], type = presorted ? TYPES[Math.floor(j * 20 / N) % 5] : TYPES[(j >> 6) % 5];
+      batch.push({ id: j + 1, account: `AC${String(10000 + (j * 7) % 9000)}`, region, type,
+        date: new Date(Date.UTC(2025, j % 12, 1 + (j % 28))), amount: Math.round(((j * 7919) % 200000 - 100000) * 1.37) / 100,
+        reference: j % 37 === 0 ? `Reversal of an earlier transfer between linked accounts, batch ${j}` : `REF-${j * 31}` });
+    }
+    yield batch;
+  }
+}
+const grouped = flag('grouped');
+const total = (label) => [{ height: 15, style: { fontWeight: 'bold' }, cells: [{ value: label }, { value: '=CountRows()' }, null, null, null, { value: '=Sum(Fields.amount)', style: { format: 'N2', textAlign: 'right' } }, null] }];
+const def = {
+  $schema: 'pagewright/report@1', name: 'Ledger', locale: 'en-IN', page: { size: 'A4', margins: [36, 36, 36, 36] },
+  // the rows come from opts.sqlRows below (a cursor); the query text is not run
+  dataSources: [{ name: 'db', type: 'sql', connection: 'ledger', query: 'select * from transactions order by id' }],
+  dataSets: [{ name: 'Tx', source: 'db', fields: [{ name: 'id', type: 'number' }, { name: 'account', type: 'string' }, { name: 'region', type: 'string' }, { name: 'type', type: 'string' },
+    { name: 'date', type: 'date' }, { name: 'amount', type: 'number' }, { name: 'reference', type: 'string' }] }],
+  sections: {
+    pageHeader: { height: 22, items: [{ type: 'textbox', name: 'Title', x: 0, y: 0, w: 300, h: 18, value: 'Transactions ledger', style: { fontWeight: 'bold', fontSize: 12 } }] },
+    body: { items: [{ type: 'table', name: 'Tx', dataSet: 'Tx', x: 0, y: 0, w: 523, h: 40,
+      columns: [{ width: 50 }, { width: 62 }, { width: 50 }, { width: 62 }, { width: 62 }, { width: 70 }, { width: 167 }],
+      header: [{ height: 16, style: { fontWeight: 'bold', backgroundColor: '#eef2f7' }, cells: ['Id', 'Account', 'Region', 'Type', 'Date', 'Amount', 'Reference'].map(value => ({ value })) }],
+      detail: [{ height: 14, style: { borderBottom: '0.5 solid #e5e7eb' }, cells: [{ value: '=Fields.id' }, { value: '=Fields.account' }, { value: '=Fields.region' }, { value: '=Fields.type' },
+        { value: '=Fields.date', style: { format: 'd' } }, { value: '=Fields.amount', style: { format: 'N2', textAlign: 'right' } }, { value: '=Fields.reference', style: { fontSize: 8 } }] }],
+      footer: total('Total'),
+      ...(grouped ? { groups: [
+        { name: 'Region', expr: '=Fields.region', repeatHeader: true, header: [{ height: 18, style: { fontWeight: 'bold', backgroundColor: '#dbeafe' }, cells: [{ value: '="Region " & Fields.region', colSpan: 3 }, null, null] }], footer: total('Region total') },
+        { name: 'Type', expr: '=Fields.type', header: [{ height: 15, cells: [null, { value: '=Fields.type' }] }], footer: total('Type total') },
+      ] } : {}) }] },
+    pageFooter: { height: 18, items: [{ type: 'textbox', name: 'Page', x: 0, y: 2, w: 523, h: 14, style: { textAlign: 'right' },
+      value: flag('plain') ? 'Page {Globals.PageNumber}' : 'Page {Globals.PageNumber} of {Globals.TotalPages}' }] },
+  },
+};
+
+let peak = 0;
+const rss = () => { const m = process.memoryUsage.rss() / 1048576; if (m > peak) peak = m; };
+rss();
+const base = peak;
+const timer = setInterval(rss, 20);
+const t0 = Date.now();
+const out = fs.createWriteStream(file);
+// --array: the rows made first, one array in the report's JSON source (how a library caller often has them)
+let arrayRssMB;
+if (flag('array') || opt('array')) {
+  const all = [];
+  for await (const b of rows()) for (const x of b) all.push(x);
+  def.dataSources = [{ name: 'db', type: 'json', data: opt('array') === 'text' ? JSON.stringify(all) : all }];
+  rss(); arrayRssMB = Math.round(process.memoryUsage.rss() / 1048576);
+}
+const r = await exportPdfStream(def, { fonts, subset, title: 'Ledger', window: Number(opt('window')) || undefined, presorted, ...(flag('memsort') ? { spill: false } : {}), sqlRows: () => rows(), maxRows: Math.max(N, 5_000_000), timeoutMs: 24 * 3600e3,
+  ...(flag('tagged') ? { tagged: { lang: 'en-IN' }, pdfa: icc ? { icc } : true } : {}), ...(opt('parallel') ? { parallel: Number(opt('parallel')) } : {}) }, out);
+await new Promise((ok, fail) => out.end((e) => (e ? fail(e) : ok())));
+clearInterval(timer);
+rss();
+console.log(JSON.stringify({ rows: N, pages: r.pages, streamed: r.streamed, ...(r.why ? { why: r.why } : {}), peakRssMB: Math.round(peak), startRssMB: Math.round(base), ...(arrayRssMB ? { arrayRssMB } : {}), ms: Date.now() - t0,
+  bytes: fs.statSync(file).size, file, node: process.version, grouped, presorted, tagged: flag('tagged'), parallel: Number(opt('parallel')) || 1 }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@reportwright/engine",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "publishConfig": {
     "access": "public"
   },
@@ -51,6 +51,7 @@
   "files": [
     "dist",
     "pool",
+    "examples",
     "index.d.ts",
     "report.d.ts",
     "schema.json",
@@ -70,7 +71,7 @@
     "bidi-js": "1.0.3"
   },
   "peerDependencies": {
-    "@reportwright/fonts": "0.11.0"
+    "@reportwright/fonts": "0.12.1"
   },
   "peerDependenciesMeta": {
     "@reportwright/fonts": {

package/pool/worker.js

@@ -10,7 +10,7 @@ const EXPORT = {
   xlsx: (m, o) => exportXlsx(m, { fonts, ...o }),
   docx: (m, o) => exportDocx(m, { fonts, ...o }),
   pptx: (m, o) => exportPptx(m, { fonts, ...o }),
-  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, fonts, o)),
+  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, { ...o, fonts })),
   csv: async (m, o) => new TextEncoder().encode(exportCsv(m, o)),
 };
 /** @type {import('node:worker_threads').MessagePort} */ (parentPort).on('message', async ({ def, opts }) => {