@reportwright/engine 0.12.0 → 0.13.0

package/dist/types/src/exporters/officestream.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Report definition → .docx written to sink as its pages are laid out.
+ * @param {any} def @param {any} opts render()'s options and exportDocx's (fontStore, fetchImage, figures, rasterize, lang,
+ *   title), plus timeoutMs or deadline, signal, window (rows laid out at once), maxRows, sorter, sqlRows as exportPdfStream
+ * @param {Sink} sink not closed
+ * @returns {Promise<StreamResult>}
+ */
+export function exportDocxStream(def: any, opts: any, sink: Sink): Promise<StreamResult>;
+/**
+ * Report definition → .xlsx written to sink as its rows come.
+ * @param {any} def @param {any} opts render()'s options and exportXlsx's (currency, title, formulas, fontStore,
+ *   fetchImage), plus timeoutMs or deadline, signal, maxRows, sorter, sqlRows as exportPdfStream
+ * @param {Sink} sink not closed
+ * @returns {Promise<StreamResult>}
+ */
+export function exportXlsxStream(def: any, opts: any, sink: Sink): Promise<StreamResult>;
+export type Sink = {
+    write(u8: Uint8Array): any;
+    once?: (ev: string, cb: () => void) => any;
+};
+export type StreamResult = {
+    streamed: boolean;
+    pages?: number;
+    rows?: number;
+    warnings: string[];
+    why?: string;
+};

package/dist/types/src/exporters/pdf.d.ts

@@ -9,8 +9,10 @@
  *   tagged?: { lang?: string }, drillBase?: string, onWarning?: (message: string) => void, timeoutMs?: number,
  *   creationDate?: Date }} [opt]
  *   subset: a HarfBuzz subsetter (see subset.js). Without it, fonts embed in full.
- *   pdfa: make a PDF/A-2b file (needs an sRGB ICC profile). prepare: last change before saving
- *   (the server adds a signature placeholder there). save: extra pdf-lib save options.
+ *   pdfa: a PDF/A-2b file (2a with tagged): true uses the sRGB profile of @reportwright/fonts, or pass { icc } (this
+ *   function itself needs icc; the engine package fills it in). prepare: last change to the pdf-lib PDFDocument before
+ *   saving (the server adds a signature placeholder there). save: pdf-lib save options. These, encrypt and PDF/A without
+ *   a profile take the whole-model path in exportPdfStream; a PDF/A file with a profile streams.
  *   encrypt: { userPassword, ownerPassword, permissions } — AES-256 (see encrypt.js); not with pdfa or prepare.
  *   tagged: an accessible PDF (PDF/UA-1, see pdfua.js): structure tree, marked content, artifacts, language, title
  *   shown; with pdfa it is PDF/A-2a. Off, the file is exactly what it was.
@@ -42,4 +44,4 @@ export function exportPdf(model: any, fontStore: import("../engine/text/fonts.js
     timeoutMs?: number;
     creationDate?: Date;
 }): any;
-import { PDFDocument } from 'pdf-lib';
+import { PDFDocument } from '../engine/lazylibs.js';

package/dist/types/src/exporters/pdfpaint.d.ts

@@ -1,14 +1,4 @@
-/**
- * Draw one run of shaped glyphs by ID, where the engine placed them: a TJ array whose numbers move each
- * glyph from where the font's own advance (the W array) leaves it to the shaper's position; marks that sit
- * higher or lower use the text rise. /ActualText gives copy and search the characters: for a left-to-right
- * run the whole run in reading order; a right-to-left run (drawn in visual order, which readers reorder
- * themselves) reads through ToUnicode, glyph by glyph, and only a cluster ToUnicode cannot spell gets a span
- * (`spans`, from glyphText); glyphs that stand for nothing (`texts`: '') go last. A span per joined letter, and
- * the dots of joined letters drawn among the letters, made pdftotext split words ("الج ميل", "الت قن ية").
- * @param {Content} c @param {Set<number>} [spans] @param {Map<number, string>} [texts]
- */
-export function drawRun(c: Content, run: any, x: any, y: any, size: any, pf: any, color: any, spans?: Set<number>, texts?: Map<number, string>): void;
+export function drawRun(c: any, run: any, x: any, y: any, size: any, pf: any, color: any, spans: any, texts: any): void;
 /**
  * The text each shaped glyph stands for, for the font's ToUnicode map: the character whose own (character-map)
  * glyph it is; the characters of its cluster that have no glyph of their own there (a conjunct, a joined
@@ -69,6 +59,8 @@ export class Content {
     /** straight lines waiting to be drawn after the text around them: { key, stroke, w, dash, segs: [x1, y1, x2, y2][] } */
     lines: any;
     tagged: boolean;
+    /** tagged, lines wait all the same and are drawn as one artifact (the streamed PDF's painter sets it) */
+    artLines: boolean;
     set s(v: string);
     /**
      * The operators so far. Any other drawing reads or appends to it, so the lines waiting are drawn first: the text

package/dist/types/src/exporters/pdfstandard.d.ts

@@ -0,0 +1,23 @@
+/** The font dictionary's entries (pdf-lib's context.obj takes them as they are: strings become names). @param {any} f a standard font */
+export function standardFontEntries(f: any): {
+    FirstChar: number;
+    LastChar: number;
+    Widths: any;
+    Encoding?: string;
+    Type: string;
+    Subtype: string;
+    BaseFont: any;
+};
+/** The same dictionary as PDF syntax. @param {any} f */
+export function standardFontDict(f: any): string;
+/** A literal string's body: one byte per character, escaped. @param {any} f a standard font @param {string} text */
+export function standardLiteral(f: any, text: string): string;
+/**
+ * A page whose standard-font text is redrawn in the embedded stand-ins (PDF/A, PDF/UA), at the standard font's glyph
+ * positions. The page and the items it changes are copies; families gets the standard families it found.
+ * @param {any} pg @param {(key: string) => any} fontOf the loaded fonts by key @param {Set<string>} families
+ */
+export function embeddedPage(pg: any, fontOf: (key: string) => any, families: Set<string>): any;
+/** The warnings for the standard families a PDF/A or PDF/UA file drew in their stand-ins (a family's name is its regular face's key). @param {Set<string>} families */
+export function embeddedWarnings(families: Set<string>): string[];
+export function standIns(store: import("../engine/text/fonts.js").FontStore): string[];

package/dist/types/src/exporters/pdfstream.d.ts

@@ -1,16 +1,16 @@
 /**
- * @typedef {{ write(u8: Uint8Array): any, once?: (ev: string, cb: () => void) => any }} Sink
- *   write may return a promise (awaited: back-pressure), or false with a 'drain' event to wait for (a Node Writable).
- * @typedef {{ title: string, page: number, top?: number, level?: number }} OutlineEntry
- *   page: 1-based; top: the destination's y in PDF space (from the bottom), else the page fits the window.
+ * p, or signal's reason as soon as it aborts (p goes on, unwatched).
+ * @template T @param {Promise<T>} p @param {AbortSignal} [signal] @returns {Promise<T>}
  */
+export function raced<T>(p: Promise<T>, signal?: AbortSignal): Promise<T>;
 /**
  * @param {Sink} sink
  * @param {{ fontStore: import('../engine/text/fonts.js').FontStore, fonts?: string[],
  *   subset?: (font: Uint8Array, codePoints: Iterable<number>, opt?: { glyphs?: Iterable<number> }) => Promise<Uint8Array>,
- *   id?: string }} opt
+ *   id?: string, signal?: AbortSignal }} opt
  *   fonts: font keys to name now (F1, F2… in this order); any other font is named on first use.
  *   subset: subset.js's subsetter; without it fonts embed in full. id: the file ID (32 hex digits), else random.
+ *   signal: stops a wait for the sink (back-pressure) when it aborts.
  */
 export function createPdfStream(sink: Sink, opt: {
     fontStore: import("../engine/text/fonts.js").FontStore;
@@ -19,6 +19,7 @@ export function createPdfStream(sink: Sink, opt: {
         glyphs?: Iterable<number>;
     }) => Promise<Uint8Array>;
     id?: string;
+    signal?: AbortSignal;
 }): {
     /** @param {string} key @returns {string} the font's resource name (F1…) */
     fontName: (key: string) => string;
@@ -124,8 +125,12 @@ export function createPdfStream(sink: Sink, opt: {
 /**
  * A report to PDF bytes written to sink as its pages are laid out (planPaged: a flowing table, one window of rows at a
  * time), else rendered whole and exported by exportPdf, the same file it always was, written to sink in one piece.
- * opts: render()'s options and exportPdf's (fontStore, subset, title, author, creationDate, drillBase…). Tagged,
- * PDF/A, encrypted and signed files take the whole-model path.
+ * opts: render()'s options and exportPdf's (fontStore, subset, title, author, creationDate, drillBase…). Tagged
+ * (PDF/UA-1) and PDF/A files stream too; encrypted and signed files, save options and PDF/A without a colour profile
+ * take the whole-model path (the result's why says which).
+ * timeoutMs (or deadline) bounds the whole export, waits for the sink included; signal (an AbortSignal) stops it at its
+ * next wait (a row batch, the sink, a sort, a page). Stopped or failed: it rejects (the signal's reason, an AbortError;
+ * a timeout: status 504, code 'ETIMEDOUT'), the rows' cursors are closed, a sort's temporary files removed.
  * @param {any} def @param {any} opts @param {Sink} sink
  * @returns {Promise<{ streamed: boolean, pages: number, warnings: string[], why?: string }>}
  */
@@ -157,12 +162,13 @@ export function paintPage(pg: any, H: number, env: any): {
 };
 /**
  * What paintPage draws with: a writer's resource names and glyph records (w), the fonts, shaped glyphs' texts.
- * @param {any} w @param {any} fontStore @param {{ drillBase?: string, pageRef?: (i: number) => any, tags?: any }} [o]
+ * @param {any} w @param {any} fontStore @param {{ drillBase?: string, pageRef?: (i: number) => any, tags?: any, embedAll?: boolean }} [o]
  */
 export function paintEnv(w: any, fontStore: any, o?: {
     drillBase?: string;
     pageRef?: (i: number) => any;
     tags?: any;
+    embedAll?: boolean;
 }): {
     shared: {
         font: (f: any) => any;
@@ -179,6 +185,8 @@ export function paintEnv(w: any, fontStore: any, o?: {
     reserve: () => any;
     drillBase: string;
     tags: any;
+    /** PDF/A, PDF/UA: standard fonts drawn in embedded stand-ins; the families that were @type {Set<string>|null} */
+    embedded: Set<string> | null;
 };
 /**
  * write may return a promise (awaited: back-pressure), or false with a 'drain' event to wait for (a Node Writable).

package/dist/types/src/exporters/pdfstreamtags.d.ts

@@ -5,11 +5,18 @@
  * ColSpan), Link > OBJR inside the element whose text it covers; page header and footer, borders and boxes artifacts.
  * Each page's rows (TR, TD), its parent-tree array and its links' elements are written when the page ends; what stays
  * is one number per row (the Table's kids) and two per page and link (the parent tree), written at the end.
+ * A cell's element is a few dozen bytes in an object stream (no /Type, which is optional; no /A on a plain TD): a
+ * million-row table has some 8 million of them, the most of the file's objects.
  */
 /**
  * @param {any} w the PDF writer (pdfstream.js createPdfStream)
+ * @param {{ maxElements?: number }} [o] maxElements: past it the export stops (413)
  */
-export function createStreamTagger(w: any): {
+export function createStreamTagger(w: any, { maxElements }?: {
+    maxElements?: number;
+}): {
+    /** structure elements so far */
+    readonly count: number;
     /** a new page (window: the paged window it came from; row ids are its own) @param {number} window */
     page(window: number): void;
     /**

package/dist/types/src/exporters/spill.d.ts

@@ -0,0 +1,20 @@
+/** At the server's start: remove the spill folders of processes that are gone (pw-spill-<pid>-…) and old-style ones. */
+export function sweepAtBoot(root?: any): void;
+export function spillSorter({ run, runBytes, frame, maxMB, totalMB, dir: root, staleMs }?: {
+    run?: number;
+    runBytes?: number;
+    frame?: number;
+    maxMB?: number;
+    totalMB?: number;
+    dir?: any;
+    staleMs?: number;
+}): (cmp: (a: any, b: any) => number) => {
+    /** @param {any} r */
+    push(r: any): Promise<void>;
+    drain(): AsyncGenerator<any, void, unknown>;
+    close(): Promise<void>;
+    /** Tests: the runs written so far. */
+    readonly runs: number;
+};
+export function canSpill(): boolean;
+export function spillShared(): any;

package/dist/types/src/exporters/subset.d.ts

@@ -1,5 +1,20 @@
+/**
+ * The tables of a single-face sfnt (TrueType or CFF) as a Map of tag to bytes. A collection (ttcf) gives an empty map.
+ * @param {Uint8Array} bytes
+ */
+export function sfntTables(bytes: Uint8Array): Map<any, any>;
+/** A sfnt from its tables: directory, 4-byte padding, table checksums and the head checkSumAdjustment. */
+export function buildSfnt(version: any, tables: any): Uint8Array<ArrayBuffer>;
+/**
+ * The name table with the given strings replaced (nameID to text). Each replaced string is written in its record's own
+ * encoding: UTF-16BE for Windows and Unicode, one byte per character for Macintosh.
+ * @param {Uint8Array} name
+ * @param {Map<number, string>} strings
+ */
+export function setNameStrings(name: Uint8Array, strings: Map<number, string>): Uint8Array<ArrayBuffer>;
 /** @param {() => Promise<ArrayBuffer|Uint8Array>} loadWasm */
 export function createSubsetter(loadWasm: () => Promise<ArrayBuffer | Uint8Array>): (font: Uint8Array, codePoints: Iterable<number>, opt?: {
     glyphs?: Iterable<number>;
     layout?: boolean;
+    weight?: number;
 }) => Promise<Uint8Array>;

package/dist/types/src/exporters/xlsxstream.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @param {{ name: string, title: string, columns: number[], timeZone: string|null, locale: string|null, currency: string|null, rows: () => AsyncGenerator<any> }} plan
+ * @param {(bytes: Uint8Array) => any} put the zip's bytes as they come (awaited: back-pressure)
+ * @param {{ currency?: string, title?: string, formulas?: boolean, maxRows?: number, timeZone?: string }} [opt]
+ */
+export function writeXlsxStream(plan: {
+    name: string;
+    title: string;
+    columns: number[];
+    timeZone: string | null;
+    locale: string | null;
+    currency: string | null;
+    rows: () => AsyncGenerator<any>;
+}, put: (bytes: Uint8Array) => any, opt?: {
+    currency?: string;
+    title?: string;
+    formulas?: boolean;
+    maxRows?: number;
+    timeZone?: string;
+}): Promise<void>;
+export const EXCEL_ROWS: 1048576;
+export const EXCEL_COLS: 16384;

package/examples/stream-1m.mjs

@@ -4,13 +4,15 @@
 // cursor), so nothing holds them all.
 //
 //   npm i @reportwright/engine @reportwright/fonts
-//   node node_modules/@reportwright/engine/examples/stream-1m.mjs [rows=1000000] [out=ledger.pdf] [--plain] [--grouped] [--tagged] [--parallel=4]
+//   node node_modules/@reportwright/engine/examples/stream-1m.mjs [rows=1000000] [out=ledger.pdf] [--plain] [--grouped] [--presorted] [--tagged] [--array[=text]] [--parallel=N]
 //
 // Prints one JSON line: { rows, pages, streamed, peakRssMB, ms, bytes, ... }. "Page N of M" is in the page footer
 // (the rows are read twice: once to count the pages); --plain leaves M out (one pass). --grouped adds two group levels
-// with subtotals; --tagged makes an accessible PDF/A (PDF/UA-1 + PDF/A-2a); --parallel=N lays out and paints with N
-// worker threads. RW_ENGINE: import the engine from another path (a checkout: packages/engine/entry.js); RW_PUBLIC: a
-// folder with fonts/ and harfbuzz-subset.wasm instead of @reportwright/fonts.
+// with subtotals (sorted through a disk spill; --presorted: the rows come in group order and are only checked); --tagged
+// makes an accessible PDF/A (PDF/UA-1 + PDF/A-2a); --array: the rows as one array in the definition (--array=text: as
+// JSON text); --parallel=N lays out and paints with N worker threads (serial by default: each thread costs 150-300 MB).
+// RW_ENGINE: import the engine from another path (a checkout: packages/engine/entry.js); RW_PUBLIC: a folder with fonts/
+// and harfbuzz-subset.wasm instead of @reportwright/fonts.
 import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
@@ -33,13 +35,16 @@ if (process.env.RW_PUBLIC) {
   icc = fs.readFileSync(path.join(dir, 'icc', 'sRGB-v2-magic.icc'));
 }
 
+const presorted = flag('presorted');
 const REGIONS = ['North', 'South', 'East', 'West'], TYPES = ['Deposit', 'Withdrawal', 'Transfer', 'Fee', 'Interest'];
 /** the rows, a thousand at a time, made as they are asked for */
 async function* rows() {
   for (let i = 0; i < N; i += 1000) {
     const batch = [];
     for (let j = i; j < Math.min(N, i + 1000); j++) {
-      batch.push({ id: j + 1, account: `AC${String(10000 + (j * 7) % 9000)}`, region: REGIONS[(j >> 9) % 4], type: TYPES[(j >> 6) % 5],
+      // --presorted: the rows come in group order (as a query ordered by region, type gives them)
+      const region = presorted ? REGIONS[Math.floor(j * 4 / N)] : REGIONS[(j >> 9) % 4], type = presorted ? TYPES[Math.floor(j * 20 / N) % 5] : TYPES[(j >> 6) % 5];
+      batch.push({ id: j + 1, account: `AC${String(10000 + (j * 7) % 9000)}`, region, type,
         date: new Date(Date.UTC(2025, j % 12, 1 + (j % 28))), amount: Math.round(((j * 7919) % 200000 - 100000) * 1.37) / 100,
         reference: j % 37 === 0 ? `Reversal of an earlier transfer between linked accounts, batch ${j}` : `REF-${j * 31}` });
     }
@@ -78,10 +83,18 @@ const base = peak;
 const timer = setInterval(rss, 20);
 const t0 = Date.now();
 const out = fs.createWriteStream(file);
-const r = await exportPdfStream(def, { fonts, subset, title: 'Ledger', sqlRows: () => rows(), maxRows: Math.max(N, 5_000_000), timeoutMs: 24 * 3600e3,
+// --array: the rows made first, one array in the report's JSON source (how a library caller often has them)
+let arrayRssMB;
+if (flag('array') || opt('array')) {
+  const all = [];
+  for await (const b of rows()) for (const x of b) all.push(x);
+  def.dataSources = [{ name: 'db', type: 'json', data: opt('array') === 'text' ? JSON.stringify(all) : all }];
+  rss(); arrayRssMB = Math.round(process.memoryUsage.rss() / 1048576);
+}
+const r = await exportPdfStream(def, { fonts, subset, title: 'Ledger', window: Number(opt('window')) || undefined, presorted, ...(flag('memsort') ? { spill: false } : {}), sqlRows: () => rows(), maxRows: Math.max(N, 5_000_000), timeoutMs: 24 * 3600e3,
   ...(flag('tagged') ? { tagged: { lang: 'en-IN' }, pdfa: icc ? { icc } : true } : {}), ...(opt('parallel') ? { parallel: Number(opt('parallel')) } : {}) }, out);
 await new Promise((ok, fail) => out.end((e) => (e ? fail(e) : ok())));
 clearInterval(timer);
 rss();
-console.log(JSON.stringify({ rows: N, pages: r.pages, streamed: r.streamed, ...(r.why ? { why: r.why } : {}), peakRssMB: Math.round(peak), startRssMB: Math.round(base), ms: Date.now() - t0,
-  bytes: fs.statSync(file).size, file, node: process.version, grouped, tagged: flag('tagged'), parallel: Number(opt('parallel')) || 1 }));
+console.log(JSON.stringify({ rows: N, pages: r.pages, streamed: r.streamed, ...(r.why ? { why: r.why } : {}), peakRssMB: Math.round(peak), startRssMB: Math.round(base), ...(arrayRssMB ? { arrayRssMB } : {}), ms: Date.now() - t0,
+  bytes: fs.statSync(file).size, file, node: process.version, grouped, presorted, tagged: flag('tagged'), parallel: Number(opt('parallel')) || 1 }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@reportwright/engine",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "publishConfig": {
     "access": "public"
   },
@@ -57,7 +57,8 @@
     "schema.json",
     "README.md",
     "CHANGELOG.md",
-    "LICENSE"
+    "LICENSE",
+    "THIRD-PARTY-NOTICES.md"
   ],
   "engines": {
     "node": ">=20.9"
@@ -66,12 +67,13 @@
   "dependencies": {
     "pdf-lib": "1.17.1",
     "@pdf-lib/fontkit": "1.1.1",
+    "@pdf-lib/standard-fonts": "1.0.0",
     "bwip-js": "4.11.4",
     "fflate": "0.8.3",
     "bidi-js": "1.0.3"
   },
   "peerDependencies": {
-    "@reportwright/fonts": "0.12.0"
+    "@reportwright/fonts": "0.13.0"
   },
   "peerDependenciesMeta": {
     "@reportwright/fonts": {

package/pool/worker.js

@@ -10,7 +10,7 @@ const EXPORT = {
   xlsx: (m, o) => exportXlsx(m, { fonts, ...o }),
   docx: (m, o) => exportDocx(m, { fonts, ...o }),
   pptx: (m, o) => exportPptx(m, { fonts, ...o }),
-  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, fonts, o)),
+  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, { ...o, fonts })),
   csv: async (m, o) => new TextEncoder().encode(exportCsv(m, o)),
 };
 /** @type {import('node:worker_threads').MessagePort} */ (parentPort).on('message', async ({ def, opts }) => {

package/report.d.ts

@@ -344,6 +344,7 @@ export interface FormField extends ItemBase {
   type: 'field';
   kind?: 'text' | 'multiline' | 'checkbox' | 'choice';
   fieldName?: Expr<string>;
+  label?: Expr<string>;
   value?: Expr;
   options?: Expr;
   required?: boolean;

package/schema.json

@@ -1259,6 +1259,9 @@
             "fieldName": {
               "x-ts": "Expr<string>"
             },
+            "label": {
+              "x-ts": "Expr<string>"
+            },
             "value": {
               "x-ts": "Expr"
             },