@reportwright/engine 0.12.0 → 0.12.1

package/dist/types/packages/engine/entry.d.ts

@@ -49,10 +49,14 @@ export function exportPptx(model: any, options?: NonNullable<Parameters<typeof e
     fonts?: FontStore;
 }, ...args: any[]): Promise<Uint8Array>;
 /**
- * @typedef {RenderOptions & Omit<PdfOptions, 'fonts'> & { fonts?: FontStore, tagged?: { lang?: string },
- *   window?: number, parallel?: number, maxRows?: number, id?: string }} PdfStreamOptions
+ * @typedef {RenderOptions & Omit<PdfOptions, 'fonts'> & { fonts?: FontStore, tagged?: { lang?: string, maxElements?: number },
+ *   window?: number, parallel?: number, maxRows?: number, id?: string, presorted?: boolean,
+ *   spill?: false | { dir?: string, maxMB?: number }, sorter?: (cmp: (a: any, b: any) => number) => any }} PdfStreamOptions
  *   render()'s options and exportPdf's, plus: window (rows laid out at once, 1000), parallel (Node: worker threads
- *   that lay out and paint pages, opt-in; the file is the same), maxRows (rows read, 5,000,000), id (the file ID).
+ *   that lay out and paint pages, opt-in; the file is the same, each thread costs memory), maxRows (rows read,
+ *   5,000,000), id (the file ID). presorted: the rows come in the report's order (its sort, its groups together), so
+ *   they are not sorted, only checked as they pass (a row out of order stops the export). spill: where a sort's runs go
+ *   (Node: the system temp folder, 2 GB at most; false sorts in memory); in a browser a sort is in memory.
  */
 /**
  * Report definition → PDF written to a sink as it is made, for reports too large to hold as a page model: a flowing
@@ -95,17 +99,27 @@ export type PdfOptions = Omit<NonNullable<Parameters<typeof exportPdfFull>[2]>,
 };
 /**
  * render()'s options and exportPdf's, plus: window (rows laid out at once, 1000), parallel (Node: worker threads
- * that lay out and paint pages, opt-in; the file is the same), maxRows (rows read, 5,000,000), id (the file ID).
+ * that lay out and paint pages, opt-in; the file is the same, each thread costs memory), maxRows (rows read,
+ * 5,000,000), id (the file ID). presorted: the rows come in the report's order (its sort, its groups together), so
+ * they are not sorted, only checked as they pass (a row out of order stops the export). spill: where a sort's runs go
+ * (Node: the system temp folder, 2 GB at most; false sorts in memory); in a browser a sort is in memory.
  */
 export type PdfStreamOptions = RenderOptions & Omit<PdfOptions, "fonts"> & {
     fonts?: FontStore;
     tagged?: {
         lang?: string;
+        maxElements?: number;
     };
     window?: number;
     parallel?: number;
     maxRows?: number;
     id?: string;
+    presorted?: boolean;
+    spill?: false | {
+        dir?: string;
+        maxMB?: number;
+    };
+    sorter?: (cmp: (a: any, b: any) => number) => any;
 };
 export type RenderOptions = import("../../src/engine/index.js").RenderOptions;
 import { FontStore } from '../../src/engine/index.js';

package/dist/types/src/engine/data/guard.d.ts

@@ -16,11 +16,14 @@ export function redirectRequest(from: URL, to: URL, req: any, status: number): a
 /**
  * The guarded default fetch (see the top of this file). In Node every hop goes through nodeRequest (address pinned);
  * in a browser through `base` with redirects by hand (literal addresses checked; a browser has no DNS to ask).
- * @param {{ allowHosts?: string[] }} opt @param {typeof fetch} base
+ * @param {{ allowHosts?: string[], viaBase?: boolean }} opt viaBase: every hop through `base`, in Node too (the browser
+ *   viewer's fetch, which its tests run in Node)
+ * @param {typeof fetch} base
  * @returns {typeof fetch}
  */
 export function guardedFetch(opt: {
     allowHosts?: string[];
+    viaBase?: boolean;
 }, base: typeof fetch): typeof fetch;
 /**
  * Any fetch with the engine's time limit and byte cap.

package/dist/types/src/engine/index.d.ts

@@ -71,7 +71,7 @@ export class ReportError extends Error {
 }
 export const BAND_ITEMS: 250000;
 /** The start of the warning for CJK text without the optional CJK fonts (the viewer opens its warnings for it). */
-export const NO_CJK_FONTS: "Text in Japanese/Chinese/Korean needs the optional CJK fonts: npm run fonts:cjk";
+export const NO_CJK_FONTS: "Text in Japanese/Chinese/Korean needs the optional CJK fonts: npm i @reportwright/fonts-cjk, then npx reportwright-fonts-cjk (npm); in the app, npm run fonts:cjk";
 /**
  * Options for render(). All optional.
  */

package/dist/types/src/engine/items/chart-kit.d.ts

@@ -10,6 +10,12 @@ export function niceScale(min: any, max: any, ticks?: number): {
     hi: number;
     step: number;
 };
+/** A nice scale whose step is the span over the ticks: niceScale rounds the span first, which can add a whole step. */
+export function niceScaleSpan(min: any, max: any, ticks?: number): {
+    lo: number;
+    hi: number;
+    step: number;
+};
 /** Ticks from lo to hi by step, rounded so 0.1 + 0.2 reads 0.3. At most 200 (a bad min/max cannot loop for ever). */
 export function linearTicks(lo: any, hi: any, step: any): any[];
 /** Powers of ten covering [min, max] (both > 0); at least two. */

package/dist/types/src/engine/stream.d.ts

@@ -1,5 +1,5 @@
 /**
- * An in-memory sorter (the browser, the CLI): the server passes one that spills runs to disk (server/spill.js).
+ * An in-memory sorter (the browser, the CLI): the server passes one that spills runs to disk (exporters/spill.js; exportPdfStream's default in Node).
  * @param {(a: any, b: any) => number} cmp
  */
 export function memorySorter(cmp: (a: any, b: any) => number): {
@@ -11,9 +11,9 @@ export function memorySorter(cmp: (a: any, b: any) => number): {
  * @param {any} def0 the report definition
  * @param {{ format: 'csv'|'xlsx', parameters?: object, timeZone?: string, now?: Date, state?: any, loadReport?: (id: string) => Promise<any>,
  *   reportId?: string, fetch?: typeof fetch, baseUrl?: string, sql?: (src: any, p: any) => Promise<object[]>,
- *   sqlRows?: (src: any, p: any) => AsyncIterable<object[]>, sources?: Record<string, any>, sorter?: (cmp: (a: any, b: any) => number) => any,
+ *   sqlRows?: (src: any, p: any) => AsyncIterable<object[]>, sources?: Record<string, any>, sorter?: (cmp: (a: any, b: any) => number) => any, presorted?: boolean,
  *   deadline?: number, timeoutMs?: number, maxRows?: number, maxJsonRowBytes?: number, maxJsonWholeBytes?: number }} opts
- * @returns {Promise<{ why: string } | { why?: undefined, name: string, columns: number[], title: string, timeZone: string|null, locale: string|null, currency: string|null, warnings: string[], rows: () => AsyncGenerator<any>, records: () => AsyncGenerator<any>, grouped: () => AsyncGenerator<any>, stats: { rows: number } }>}
+ * @returns {Promise<{ why: string } | { why?: undefined, name: string, columns: number[], title: string, timeZone: string|null, locale: string|null, currency: string|null, warnings: string[], rows: () => AsyncGenerator<any>, records: () => AsyncGenerator<any>, grouped: () => AsyncGenerator<any>, fieldList: () => Promise<any[]>, stats: { rows: number } }>}
  */
 export function planStream(def0: any, opts: {
     format: "csv" | "xlsx";
@@ -29,6 +29,7 @@ export function planStream(def0: any, opts: {
     sqlRows?: (src: any, p: any) => AsyncIterable<object[]>;
     sources?: Record<string, any>;
     sorter?: (cmp: (a: any, b: any) => number) => any;
+    presorted?: boolean;
     deadline?: number;
     timeoutMs?: number;
     maxRows?: number;
@@ -48,6 +49,7 @@ export function planStream(def0: any, opts: {
     rows: () => AsyncGenerator<any>;
     records: () => AsyncGenerator<any>;
     grouped: () => AsyncGenerator<any>;
+    fieldList: () => Promise<any[]>;
     stats: {
         rows: number;
     };

package/dist/types/src/engine/text/measure.d.ts

@@ -19,6 +19,8 @@ export function saBreaks(text: string, segmenter?: Intl.Segmenter | null): Set<n
  * @param {any[]} items @param {TextMeasurer} m
  */
 export function finishText(items: any[], m: TextMeasurer): void;
+/** Unicode format characters (Cf): bidi marks and embeddings, joiners, soft hyphen. They have no shape of their own. */
+export const FORMAT_CHAR: RegExp;
 /**
  * @typedef {{ font: string, x: number, w: number, text: string, rtl?: boolean, shaped?: boolean, glyphs: number[] }} Run
  *   One font, one direction. x and w in points from the start of the line; glyphs: [id, x, y, cluster] per glyph

package/dist/types/src/exporters/docx.d.ts

@@ -1,9 +1,12 @@
+/** The string cut into pieces of at most about max chars, each ending at a '>' (a tag boundary), so their join is s. */
+export function slices(s: any, max: any): any[];
 /** Left, right or centre, from where the lines sit in their box. */
 export function alignOf(it: any): "center" | "left" | "right" | "both";
 /**
  * @param {any} model render() output
  * @param {{ title?: string, author?: string, lang?: string, fontStore?: any, fetchImage?: (src: string) => Promise<Uint8Array|null>,
- *   rasterize?: (svg: string, w: number, h: number, scale: number) => Promise<Uint8Array|null>, figures?: boolean }} [opt]
+ *   rasterize?: (svg: string, w: number, h: number, scale: number) => Promise<Uint8Array|null>, figures?: boolean, xmlChunk?: number }} [opt]
+ *   xmlChunk: the document XML is written to the zip in pieces of about this many chars (default 4M; tests lower it)
  *   fontStore: text in pictures as outlines (else the reader's fallback font); fetchImage: pictures by URL;
  *   rasterize: the PNG fallback of a picture (default: a canvas in a browser, sharp in Node); figures: false leaves them out.
  * @returns {Promise<Uint8Array>}
@@ -16,4 +19,5 @@ export function exportDocx(model: any, opt?: {
     fetchImage?: (src: string) => Promise<Uint8Array | null>;
     rasterize?: (svg: string, w: number, h: number, scale: number) => Promise<Uint8Array | null>;
     figures?: boolean;
+    xmlChunk?: number;
 }): Promise<Uint8Array>;

package/dist/types/src/exporters/html.d.ts

@@ -4,12 +4,15 @@
  * drill-through opens the ReportWright viewer at drillBase (without one it is shown disabled).
  * Unless static, a file whose report has tables also gets a Tables view (htmldata.js): the tables as HTML, re-sorted on
  * their interactive-sort columns, groups opened and closed, rows filtered, by one inline script the file's CSP allows.
+ * exportHtml(model, { fonts, ...options }). The 0.7 form exportHtml(model, fontStore, options) still works (deprecated).
  * @param {any} model
- * @param {import('../engine/text/fonts.js').FontStore} fontStore
- * @param {{ title?: string, drillBase?: string, subset?: (font: Uint8Array, cps: Iterable<number>, o?: { layout?: boolean }) => Promise<Uint8Array>,
- *   static?: boolean, locale?: string, currency?: string, lang?: string, strings?: Partial<typeof DATA_STRINGS> }} [opt]
+ * @param {{ fonts?: import('../engine/text/fonts.js').FontStore, fontStore?: import('../engine/text/fonts.js').FontStore, title?: string, drillBase?: string, subset?: (font: Uint8Array, cps: Iterable<number>, o?: { layout?: boolean }) => Promise<Uint8Array>,
+ *   static?: boolean, locale?: string, currency?: string, lang?: string, strings?: Partial<typeof DATA_STRINGS>, timeoutMs?: number } | import('../engine/text/fonts.js').FontStore} [options]
+ * @param {any} [legacyOpt] the options of the 0.7 form
  */
-export function exportHtml(model: any, fontStore: import("../engine/text/fonts.js").FontStore, opt?: {
+export function exportHtml(model: any, options?: {
+    fonts?: import("../engine/text/fonts.js").FontStore;
+    fontStore?: import("../engine/text/fonts.js").FontStore;
     title?: string;
     drillBase?: string;
     subset?: (font: Uint8Array, cps: Iterable<number>, o?: {
@@ -20,5 +23,6 @@ export function exportHtml(model: any, fontStore: import("../engine/text/fonts.j
     currency?: string;
     lang?: string;
     strings?: Partial<typeof DATA_STRINGS>;
-}): any;
+    timeoutMs?: number;
+} | import("../engine/text/fonts.js").FontStore, legacyOpt?: any): Promise<string>;
 import { DATA_STRINGS } from './htmldata.js';

package/dist/types/src/exporters/pdfpaint.d.ts

@@ -1,14 +1,4 @@
-/**
- * Draw one run of shaped glyphs by ID, where the engine placed them: a TJ array whose numbers move each
- * glyph from where the font's own advance (the W array) leaves it to the shaper's position; marks that sit
- * higher or lower use the text rise. /ActualText gives copy and search the characters: for a left-to-right
- * run the whole run in reading order; a right-to-left run (drawn in visual order, which readers reorder
- * themselves) reads through ToUnicode, glyph by glyph, and only a cluster ToUnicode cannot spell gets a span
- * (`spans`, from glyphText); glyphs that stand for nothing (`texts`: '') go last. A span per joined letter, and
- * the dots of joined letters drawn among the letters, made pdftotext split words ("الج ميل", "الت قن ية").
- * @param {Content} c @param {Set<number>} [spans] @param {Map<number, string>} [texts]
- */
-export function drawRun(c: Content, run: any, x: any, y: any, size: any, pf: any, color: any, spans?: Set<number>, texts?: Map<number, string>): void;
+export function drawRun(c: any, run: any, x: any, y: any, size: any, pf: any, color: any, spans: any, texts: any): void;
 /**
  * The text each shaped glyph stands for, for the font's ToUnicode map: the character whose own (character-map)
  * glyph it is; the characters of its cluster that have no glyph of their own there (a conjunct, a joined
@@ -69,6 +59,8 @@ export class Content {
     /** straight lines waiting to be drawn after the text around them: { key, stroke, w, dash, segs: [x1, y1, x2, y2][] } */
     lines: any;
     tagged: boolean;
+    /** tagged, lines wait all the same and are drawn as one artifact (the streamed PDF's painter sets it) */
+    artLines: boolean;
     set s(v: string);
     /**
      * The operators so far. Any other drawing reads or appends to it, so the lines waiting are drawn first: the text

package/dist/types/src/exporters/pdfstream.d.ts

@@ -124,8 +124,9 @@ export function createPdfStream(sink: Sink, opt: {
 /**
  * A report to PDF bytes written to sink as its pages are laid out (planPaged: a flowing table, one window of rows at a
  * time), else rendered whole and exported by exportPdf, the same file it always was, written to sink in one piece.
- * opts: render()'s options and exportPdf's (fontStore, subset, title, author, creationDate, drillBase…). Tagged,
- * PDF/A, encrypted and signed files take the whole-model path.
+ * opts: render()'s options and exportPdf's (fontStore, subset, title, author, creationDate, drillBase…). Tagged
+ * (PDF/UA-1) and PDF/A files stream too; encrypted and signed files, save options and PDF/A without a colour profile
+ * take the whole-model path (the result's why says which).
  * @param {any} def @param {any} opts @param {Sink} sink
  * @returns {Promise<{ streamed: boolean, pages: number, warnings: string[], why?: string }>}
  */

package/dist/types/src/exporters/pdfstreamtags.d.ts

@@ -5,11 +5,18 @@
  * ColSpan), Link > OBJR inside the element whose text it covers; page header and footer, borders and boxes artifacts.
  * Each page's rows (TR, TD), its parent-tree array and its links' elements are written when the page ends; what stays
  * is one number per row (the Table's kids) and two per page and link (the parent tree), written at the end.
+ * A cell's element is a few dozen bytes in an object stream (no /Type, which is optional; no /A on a plain TD): a
+ * million-row table has some 8 million of them, the most of the file's objects.
  */
 /**
  * @param {any} w the PDF writer (pdfstream.js createPdfStream)
+ * @param {{ maxElements?: number }} [o] maxElements: past it the export stops (413)
  */
-export function createStreamTagger(w: any): {
+export function createStreamTagger(w: any, { maxElements }?: {
+    maxElements?: number;
+}): {
+    /** structure elements so far */
+    readonly count: number;
     /** a new page (window: the paged window it came from; row ids are its own) @param {number} window */
     page(window: number): void;
     /**

package/dist/types/src/exporters/spill.d.ts

@@ -0,0 +1,20 @@
+/** At the server's start: remove the spill folders of processes that are gone (pw-spill-<pid>-…) and old-style ones. */
+export function sweepAtBoot(root?: any): void;
+export function spillSorter({ run, runBytes, frame, maxMB, totalMB, dir: root, staleMs }?: {
+    run?: number;
+    runBytes?: number;
+    frame?: number;
+    maxMB?: number;
+    totalMB?: number;
+    dir?: any;
+    staleMs?: number;
+}): (cmp: (a: any, b: any) => number) => {
+    /** @param {any} r */
+    push(r: any): Promise<void>;
+    drain(): AsyncGenerator<any, void, unknown>;
+    close(): Promise<void>;
+    /** Tests: the runs written so far. */
+    readonly runs: number;
+};
+export function canSpill(): boolean;
+export function spillShared(): any;

package/dist/types/src/exporters/subset.d.ts

@@ -2,4 +2,5 @@
 export function createSubsetter(loadWasm: () => Promise<ArrayBuffer | Uint8Array>): (font: Uint8Array, codePoints: Iterable<number>, opt?: {
     glyphs?: Iterable<number>;
     layout?: boolean;
+    weight?: number;
 }) => Promise<Uint8Array>;

package/examples/stream-1m.mjs

@@ -4,13 +4,15 @@
 // cursor), so nothing holds them all.
 //
 //   npm i @reportwright/engine @reportwright/fonts
-//   node node_modules/@reportwright/engine/examples/stream-1m.mjs [rows=1000000] [out=ledger.pdf] [--plain] [--grouped] [--tagged] [--parallel=4]
+//   node node_modules/@reportwright/engine/examples/stream-1m.mjs [rows=1000000] [out=ledger.pdf] [--plain] [--grouped] [--presorted] [--tagged] [--array[=text]] [--parallel=N]
 //
 // Prints one JSON line: { rows, pages, streamed, peakRssMB, ms, bytes, ... }. "Page N of M" is in the page footer
 // (the rows are read twice: once to count the pages); --plain leaves M out (one pass). --grouped adds two group levels
-// with subtotals; --tagged makes an accessible PDF/A (PDF/UA-1 + PDF/A-2a); --parallel=N lays out and paints with N
-// worker threads. RW_ENGINE: import the engine from another path (a checkout: packages/engine/entry.js); RW_PUBLIC: a
-// folder with fonts/ and harfbuzz-subset.wasm instead of @reportwright/fonts.
+// with subtotals (sorted through a disk spill; --presorted: the rows come in group order and are only checked); --tagged
+// makes an accessible PDF/A (PDF/UA-1 + PDF/A-2a); --array: the rows as one array in the definition (--array=text: as
+// JSON text); --parallel=N lays out and paints with N worker threads (serial by default: each thread costs 150-300 MB).
+// RW_ENGINE: import the engine from another path (a checkout: packages/engine/entry.js); RW_PUBLIC: a folder with fonts/
+// and harfbuzz-subset.wasm instead of @reportwright/fonts.
 import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
@@ -33,13 +35,16 @@ if (process.env.RW_PUBLIC) {
   icc = fs.readFileSync(path.join(dir, 'icc', 'sRGB-v2-magic.icc'));
 }
 
+const presorted = flag('presorted');
 const REGIONS = ['North', 'South', 'East', 'West'], TYPES = ['Deposit', 'Withdrawal', 'Transfer', 'Fee', 'Interest'];
 /** the rows, a thousand at a time, made as they are asked for */
 async function* rows() {
   for (let i = 0; i < N; i += 1000) {
     const batch = [];
     for (let j = i; j < Math.min(N, i + 1000); j++) {
-      batch.push({ id: j + 1, account: `AC${String(10000 + (j * 7) % 9000)}`, region: REGIONS[(j >> 9) % 4], type: TYPES[(j >> 6) % 5],
+      // --presorted: the rows come in group order (as a query ordered by region, type gives them)
+      const region = presorted ? REGIONS[Math.floor(j * 4 / N)] : REGIONS[(j >> 9) % 4], type = presorted ? TYPES[Math.floor(j * 20 / N) % 5] : TYPES[(j >> 6) % 5];
+      batch.push({ id: j + 1, account: `AC${String(10000 + (j * 7) % 9000)}`, region, type,
         date: new Date(Date.UTC(2025, j % 12, 1 + (j % 28))), amount: Math.round(((j * 7919) % 200000 - 100000) * 1.37) / 100,
         reference: j % 37 === 0 ? `Reversal of an earlier transfer between linked accounts, batch ${j}` : `REF-${j * 31}` });
     }
@@ -78,10 +83,18 @@ const base = peak;
 const timer = setInterval(rss, 20);
 const t0 = Date.now();
 const out = fs.createWriteStream(file);
-const r = await exportPdfStream(def, { fonts, subset, title: 'Ledger', sqlRows: () => rows(), maxRows: Math.max(N, 5_000_000), timeoutMs: 24 * 3600e3,
+// --array: the rows made first, one array in the report's JSON source (how a library caller often has them)
+let arrayRssMB;
+if (flag('array') || opt('array')) {
+  const all = [];
+  for await (const b of rows()) for (const x of b) all.push(x);
+  def.dataSources = [{ name: 'db', type: 'json', data: opt('array') === 'text' ? JSON.stringify(all) : all }];
+  rss(); arrayRssMB = Math.round(process.memoryUsage.rss() / 1048576);
+}
+const r = await exportPdfStream(def, { fonts, subset, title: 'Ledger', window: Number(opt('window')) || undefined, presorted, ...(flag('memsort') ? { spill: false } : {}), sqlRows: () => rows(), maxRows: Math.max(N, 5_000_000), timeoutMs: 24 * 3600e3,
   ...(flag('tagged') ? { tagged: { lang: 'en-IN' }, pdfa: icc ? { icc } : true } : {}), ...(opt('parallel') ? { parallel: Number(opt('parallel')) } : {}) }, out);
 await new Promise((ok, fail) => out.end((e) => (e ? fail(e) : ok())));
 clearInterval(timer);
 rss();
-console.log(JSON.stringify({ rows: N, pages: r.pages, streamed: r.streamed, ...(r.why ? { why: r.why } : {}), peakRssMB: Math.round(peak), startRssMB: Math.round(base), ms: Date.now() - t0,
-  bytes: fs.statSync(file).size, file, node: process.version, grouped, tagged: flag('tagged'), parallel: Number(opt('parallel')) || 1 }));
+console.log(JSON.stringify({ rows: N, pages: r.pages, streamed: r.streamed, ...(r.why ? { why: r.why } : {}), peakRssMB: Math.round(peak), startRssMB: Math.round(base), ...(arrayRssMB ? { arrayRssMB } : {}), ms: Date.now() - t0,
+  bytes: fs.statSync(file).size, file, node: process.version, grouped, presorted, tagged: flag('tagged'), parallel: Number(opt('parallel')) || 1 }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@reportwright/engine",
-  "version": "0.12.0",
+  "version": "0.12.1",
   "publishConfig": {
     "access": "public"
   },
@@ -71,7 +71,7 @@
     "bidi-js": "1.0.3"
   },
   "peerDependencies": {
-    "@reportwright/fonts": "0.12.0"
+    "@reportwright/fonts": "0.12.1"
   },
   "peerDependenciesMeta": {
     "@reportwright/fonts": {

package/pool/worker.js

@@ -10,7 +10,7 @@ const EXPORT = {
   xlsx: (m, o) => exportXlsx(m, { fonts, ...o }),
   docx: (m, o) => exportDocx(m, { fonts, ...o }),
   pptx: (m, o) => exportPptx(m, { fonts, ...o }),
-  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, fonts, o)),
+  html: async (m, o) => new TextEncoder().encode(await exportHtml(m, { ...o, fonts })),
   csv: async (m, o) => new TextEncoder().encode(exportCsv(m, o)),
 };
 /** @type {import('node:worker_threads').MessagePort} */ (parentPort).on('message', async ({ def, opts }) => {