@xeplr/actions 1.1.17 → 1.1.19

package/README.md

@@ -107,6 +107,8 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
 | key | action name | does |
 |---|---|---|
 | `dataOperation` | `data-operation` | From a list of rows, make the list the next step needs — keep some rows, pick and rename columns, compute new ones, sort, optionally render. See [Data operation](#data-operation). |
+| `exportTable` | `export-table` | Rows to a **CSV, Excel (.xlsx) or PDF** file. See [Export](#export). |
+| `exportPages` | `export-pages` | Pages of placed blocks (text, tables, images, shapes) to a PDF — a dashboard as it looked. See [Export](#export). |
 | `dbFetch` | `db-fetch` | Stream rows from a table (`mode: 'table'`) or SQL (`mode: 'query'`, `params`). Default `streaming_mode: true` → NDJSON file `{ filePath, format: 'jsonl', bytes, rows }`; `false` → `{ rows, rowCount }`. |
 | `dbPush` | `db-push` | Write `rows` or an NDJSON `filePath` into `targetTable` through the uploader. `primaryKeys` → upsert. |
 | `dbMove` | `db-move` | One window of rows from a source table, query or procedure into a target table in one streamed pass. See [db-move](#db-move). |
@@ -114,8 +116,8 @@ Register the ones you want (`register(actions.builtins.dbFetch)`) or pass a modu
 | `dbListTables` | `db-list-tables` | `{ tables }` — base tables in a schema/database. |
 | `dbListViews` | `db-list-views` | `{ views }`. |
 | `dbListProcedures` | `db-list-procedures` | `{ procedures }` (functions excluded). |
-| `dbListColumns` | `db-list-columns` | `{ columns }` — name and data type for one table. |
-| `fileUpload` | `file-upload` | Read a file (`sourceType`, `sourcePath`), parse it (`format`), load it into `targetTable` (`dbType`, `dbConnection`). |
+| `dbListColumns` | `db-list-columns` | `{ columns }` — `name`, `dataType`, `udtName` and `maxLength` (characters; `-1` is SQL Server `(max)`, `null` when it has none) for one table. |
+| `fileUpload` | `file-upload` | Read a file (`sourceType`, `sourcePath`), parse it (`format`), load it into `targetTable` (`dbType`, `dbConnection`). Passes `system.onProgress` (`{ rowsRead, batches, rowsWritten }`, per batch) and `system.signal` (stop) to the uploader, as `db-move` does. |
 | `sendEmail` | `email-send` | Send one message. See [Email](#email). |
 | `emailRead` | `email-read` | List messages, newest first, with `unread_only`, `limit` (1–1000, default 50), `since`, `from_equals`, `from_contains`, `subject_contains`, `body: 'full' \| 'text' \| 'none'`, optional NDJSON spooling. |
 | `emailMove` | `email-move` | Move a message (`messageId`, `folder`) to `toFolder`. |
@@ -131,7 +133,7 @@ The db actions `db-fetch`, `db-push` and `db-procedure` accept a connection two
 - `connectionInfoId` (+ `dbInfoId`) — ids of a saved connection. The action does not resolve these; a host does, before `runAction` (e.g. `@xeplr/jobs` `resolveConnection`, which turns `connectionInfoId` into `connection` and `dbType`).
 - `connection: { host, port, user, password, database }` and `dbType` — literal, for scripts, tests and callers with no host.
 
-`db-move` takes literal `sourceConnection` / `targetConnection`; `file-upload` takes `dbConnection`. A DuckDB connection is `{ file, access?: 'ro' | 'rw', openTimeoutMs?: 60000, maxConnections?: 4 }`.
+`db-move` takes literal `sourceConnection` / `targetConnection`; `file-upload` takes `dbConnection`. A DuckDB connection is `{ file, access?: 'ro' | 'rw', openTimeoutMs?: 60000, maxConnections?: 4, keepIndexes?: false }` (see [A written DuckDB file keeps no index](#a-written-duckdb-file-keeps-no-index)).
 
 ### db-move
 
@@ -142,7 +144,8 @@ The db actions `db-fetch`, `db-push` and `db-procedure` accept a connection two
 | `writeMode` | `append` (default), `replace` (TRUNCATE the target first if it exists, DELETE if TRUNCATE fails), `upsert` (**requires** `primaryKeys`). |
 | `window` | `{ column, from, to }` → `column >= from AND column < to` (a null edge is left out). `{ columns: [{ column, from }] }` → OR of `column >= from` (null `from`s left out). For a procedure: `{ from, to, fromParam, toParam }`, passed as parameters. A window with none of `column`, `columns`, `fromParam`, `toParam` is treated as no window. |
 | `where` | Raw SQL condition, ANDed with the window in parentheses, run on the source. Not parameterised. Ignored for `mode: 'procedure'`. |
-| `staged` | `true`: write into `<targetTable>__xeplr_staging` and move it into the target in one transaction at the end (see [Staged loads](#staged-loads)). DuckDB targets only. Default `false`. |
+| `replaceFrom` | `{ column }` with `staged`: the final move deletes target rows from the earliest staged value of `column` on, before inserting the staged ones — see [Staged loads](#staged-loads). |
+| `staged` | `true`: write into `<targetTable>__xeplr_staging` and move it into the target in one transaction at the end (see [Staged loads](#staged-loads)). DuckDB targets only. Default `false`, except that **every DuckDB upsert is staged** whatever this says. |
 | `movementId` | Correlation key. Default `system.occurrenceId`, else `mv_<id>`. |
 | `batchSize`, `concurrency` | Default 5000 and 4. |
 
@@ -219,6 +222,48 @@ should not take the other eight hundred and ninety-nine with it.
 **No aggregation.** `group by status, count` changes what a row *is*; that is
 a different shape and, when it is wanted, a different step.
 
+## Export
+
+`export-table` writes rows that arrive **already presented**: which columns, in what order, under what heading, and for each cell its raw value and the text a reader saw. The action knows nothing of reports or pivots; the caller decides all that (in xeplr-bi, `report-engine`'s `presentTable` does). Each format takes what it needs:
+
+| format | writes | from |
+|---|---|---|
+| `csv` | RFC 4180; a UTF-8 byte-order mark so Excel reads `₹` and accents; a text cell starting with `= + - @` gets a leading `'` (no formula injection; numbers untouched) | `values` |
+| `xlsx` | numbers and dates as **real typed cells carrying the column's Excel number format** (`"₹"#,##0.00`, `0.0%`, `mmm yyyy`, `#,##0.0,,"M"`), so the sheet looks like the screen and still sums; an optional title and subtitle; a bold, frozen header with an autofilter; bold subtotals and a tinted total row; column widths from the first 200 rows. Excel's 1,048,576-row limit is respected (`truncated`) | `values` |
+| `pdf` | each cell exactly as displayed, numbers right-aligned by `align`, header repeated on every page, landscape past 6 columns, "Page x of y", bold or tinted total rows. Capped at 5,000 rows (`maxRows`) with a closing note | `display` |
+
+```js
+await runAction(builtins.exportTable, {
+  format: 'xlsx',
+  document: {
+    title: 'Sales by region', subtitle: 'Date: Current financial year',
+    columns: [
+      { header: 'Region' },
+      { header: 'Month', kind: 'date', numFmt: 'mmm yyyy' },
+      { header: 'Revenue', kind: 'number', numFmt: '"₹"#,##0.00', align: 'right' }
+    ]
+  },
+  rows: [{ values: ['North', '2024-01-01', 1234.5], display: ['North', 'Jan 2024', '₹1,234.50'], kind: 'detail' }]
+  // or rowsFile: an NDJSON file, one row per line — streamed, never held
+})
+// → { filePath, fileName, contentType, bytes, rows, truncated }
+```
+
+- **Streaming.** CSV and Excel stream rows to disk, from `rows` or from `rowsFile` (NDJSON, one row per line, as `streaming.spool` writes it); 200,000 rows to xlsx take about 2 s in flat memory. Our own xlsx writer runs on `fflate`. `exceljs` was rejected: it pulls a `uuid` with a published advisory and has had no release since December 2024.
+- **Fonts.** pdfkit's built-in fonts cannot draw `₹` or non-Latin text; a missing glyph draws as nothing. The PDF embeds **DejaVu Sans Condensed** (the `dejavu-fonts-ttf` peer). Pass `document.font` / `boldFont` (TTF/OTF paths) for scripts it lacks. Only the glyphs used are embedded.
+- **Output** goes to `outputPath`, or to a new directory under `XEPLR_ACTIONS_TMP_DIR` with a file named after the title. A failed export leaves no half-written file.
+
+`export-pages` lays out `pages: [{ blocks }]`, each block positioned by **fractions** of the page's content box (A4 landscape by default):
+- `text` (`text`, `size`, `bold`, `color`, `align`)
+- `table` (`title`, `columns`, `rows`; drawn inside its box, and rows that don't fit are counted as "+N more rows")
+- `image` (`src` must be a **PNG or JPEG data URL**: no fetching, no SVG, no markup)
+- `rect` (`fill`, `stroke`, `radius`)
+- `line`
+
+What it is given is assumed to come from a browser, so it is bounded: 50 pages, 200 blocks a page, 8 MB per image, 500 table rows, and 5,000 characters of text. An image that will not decode draws a note in its box instead of failing the whole file.
+
+Peers (optional, needed only by the format used): `fflate` for xlsx; `pdfkit` and `dejavu-fonts-ttf` for pdf. `writers` (`getWriter`, `INFO`, `SUPPORTED`) is exported for code that wants a writer without the action.
+
 ## Uploader
 
 `uploader.upload(opts)` is what `db-push`, `db-move` and `file-upload` use:
@@ -245,7 +290,7 @@ await actions.uploader.upload({
 
 `signal` (an `AbortSignal`, passed to db-move as `system.signal`) stops a running movement: the writes still queued are dropped at once (in-flight ones settle), the read stops at the next batch, and it ends with an error whose `code` is `'STOPPED'` (the runner keeps `error.code`), neither a failure nor a success. What it wrote stays, under its `movementId`, for the caller to roll back or keep.
 
-Rows are spooled to NDJSON files on disk between source and database. From the first batch it creates, if missing: the target table, `<targetTable>_import_errors` (`movement_id`, `row_num`, `error_description`, `underlying_sql`, `raw_row`, `recorded_at`), a `__xeplr_movement_id__` column, and a unique index on `primaryKeys`. Server dialects also add `__xeplr_id__`; DuckDB does not.
+Rows are spooled to NDJSON files on disk between source and database. From the first batch it creates, if missing: the target table, `<targetTable>_import_errors` (`movement_id`, `row_num`, `error_description`, `underlying_sql`, `raw_row`, `recorded_at`), a `__xeplr_movement_id__` column, and a unique index on `primaryKeys` (not on DuckDB: its upserts are staged and keep none). Server dialects also add `__xeplr_id__`; DuckDB does not.
 
 Logical types and what each driver creates:
 
@@ -268,9 +313,22 @@ With `staged: true` a load **never writes into its target while it runs**. Batch
 - **Crash during the move**: the transaction is discarded on reopen and the target is as it was.
 - **Did it land?** `uploader.wasCommitted({ driver, connection, movementId })` reads the ledger row, written in the same transaction as the move: a row means it landed, `null` means nothing did. No scan of the target.
 - **Upsert** replaces target rows whose key is staged (`DELETE … USING` staging, then `INSERT`). Within one load the last row staged for a key wins, as under `ON CONFLICT DO UPDATE`; rows with a NULL key match nothing and are kept. **No unique index**: a staged load drops `<targetTable>_upsert_uniq` and never creates it. DuckDB's DELETE through an index that a killed load left out of step with its rows fails with "Failed to delete all rows from index", and that failure invalidates the whole database.
+- **A range** (`replaceFrom: { column }`, staged append only, no `primaryKeys`) replaces the target's rows from **the earliest staged value** of `column` on by the staged ones, in the same one transaction: *re-load the last N days*, for a table with no key or a date with no time, where reading "past the latest" would duplicate today on every run. The caller reads the source from its cutoff; the boundary is taken from what that read brought, never from a date passed in, because a cutoff computed elsewhere sits a time zone away (a `DATE` read as UTC midnight here, compared at local midnight at the source) and a day deleted but not read back is lost. Nothing staged deletes nothing. `merged.replaced` is the rows deleted.
 - **Cost** is one statement at the end. Measured with 50 lakh staged rows into 50 crore: append 0.3 s (independent of the target's size), upsert 0.4–7 s depending on how widely the replaced keys are spread.
 
-A driver opts in by implementing `prepareStaging`, `mergeStaged`, `dropStaging` and `committedMovement`; `upload` refuses `staged` for one that does not.
+A driver opts in by implementing `prepareStaging`, `mergeStaged`, `dropStaging` and `committedMovement`; `upload` refuses `staged` for one that does not. A driver that sets `stagesUpserts: true` (DuckDB) has every upsert staged, asked or not.
+
+### A written DuckDB file keeps no index
+
+DuckDB 1.5.4–1.5.6 has a bug ([duckdb/duckdb#26106](https://github.com/duckdb/duckdb/issues/26106), open upstream): when a process dies with rows still only in the WAL, the next open replays them into the table and the next checkpoint drops them from every index on it. The first `DELETE` that reaches one fails with *"Failed to delete all rows from index"*, which is **FATAL**: the whole file is unusable until the process restarts. Any index, unique or not. A table with no index is unaffected.
+
+So the DuckDB driver, on every **read-write** open of a file:
+
+1. runs `CHECKPOINT` first, which folds a replayed WAL in safely before anything touches it (the upstream workaround);
+2. **drops every index** in the file, logging which, so one left from before this rule cannot be what a later crash breaks;
+3. **refuses `CREATE INDEX`** on that pool from then on (`ensureUpsertIndex` included).
+
+Upserts need no index: they are staged and match keys in the move. A read-only open changes nothing. `keepIndexes: true` turns 2 and 3 off, only for a file that is never written after its index exists; `:memory:` is exempt, having no WAL to replay. `test/duckdb-index-guard.test.js` kills a real writer process to bring the failure about, proves it still happens on the pinned DuckDB, and proves each step above prevents it.
 
 `uploader.rollback({ movementId, driver, connection, targetTable, metaStore?, queue? })` aborts the movement's queued work, deletes its rows from the target and error tables by `__xeplr_movement_id__` / `movement_id`, and records `rolled-back`.
 
@@ -380,7 +438,7 @@ node --test test/uploader/reconcile.test.js
 | needs nothing | needs a database |
 |---|---|
 | `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
-| `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js`, `test/staged-load.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
+| `test/list-columns-length.test.js` (`XEPLR_TEST_PG_URL`, `MYSQL_TEST_URL`, `MSSQL_TEST_URL`; each skipped when unset), `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js`, `test/duckdb-index-guard.test.js`, `test/staged-load.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
 | | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
 | | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |
 

package/index.js

@@ -41,6 +41,8 @@ module.exports = {
   streaming:     streaming,
   drivers:       { db: dbDrivers, file: fileDrivers },
   formats:       formats,
+  // { getWriter(type), INFO, SUPPORTED } — csv / xlsx / pdf writers behind export-table.
+  writers:       require('./lib/formats/writers'),
 
   // xeplr_configs — shared control-plane DB (import/movement metadata).
   // One call per app: attachConfig({ service }); await xcfg.ready(). See

package/lib/builtins/db/move.js

@@ -99,6 +99,12 @@ module.exports = {
                    'the end, so a movement that stops or fails midway never touches the target. ' +
                    'DuckDB targets only.' },
 
+    { name: 'replaceFrom',     type: 'object', order: 12.6, group: 'Write options',
+      description: 'With staged and append: { column }. The final move first deletes the target ' +
+                   'rows where column >= the earliest staged value, then inserts the staged rows, in the ' +
+                   'same one transaction — "re-load the last N days" for a table with no key, or a date ' +
+                   'with no time. Read the source from the cutoff; nothing staged deletes nothing.' },
+
     { name: 'window',          type: 'object', order: 13, group: 'Advanced',
       description: 'One window: { column, from, to }. Half-open — from <= x < to. ' +
                    'Omitted = no filter. For mode="procedure", also needs { fromParam, toParam }.' },
@@ -315,7 +321,8 @@ module.exports = {
         onProgress:     system.onProgress || null,
         // An AbortSignal from the caller: stop reading, drop the queued writes.
         signal:         system.signal || null,
-        staged:         Boolean(input.staged)
+        staged:         Boolean(input.staged),
+        replaceFrom:    input.replaceFrom || null
       });
 
       // Every count upload() has, not just the headline one — "read" and

package/lib/builtins/export/pages.js

@@ -0,0 +1,92 @@
+// export-pages — placed blocks to a PDF: a dashboard as it looked.
+//
+// Each page is a list of blocks positioned by FRACTIONS of the page's content
+// box (x, w across; y, h down), so a layout drawn on any screen keeps its
+// proportions on paper. A block is text, a small table, an image (a PNG or
+// JPEG data URL — a chart is captured as one), a rectangle or a line. See
+// lib/formats/writers/pdf.js renderPages for each block's fields.
+//
+// Bounded on purpose: what arrives here was posted by a browser, so the page,
+// block and image counts and sizes are capped and an image must be a data URL
+// of a raster type — no fetching, no markup.
+
+var fs = require('fs');
+var os = require('os');
+var path = require('path');
+var pdf = require('../../formats/writers/pdf');
+var { checkFormatRequires } = require('../../formats');
+
+var LIMITS = { pages: 50, blocksPerPage: 200, imageBytes: 8 * 1024 * 1024, tableRows: 500, text: 5000 };
+var TYPES = ['text', 'table', 'image', 'rect', 'line'];
+
+function fail(message) {
+  var err = new Error(message);
+  err.status = 400;
+  return err;
+}
+
+function fraction(n) { return typeof n === 'number' && isFinite(n) && n >= -0.01 && n <= 50; }
+
+/** Throws (400) naming the first block that is not one this action draws. */
+function validate(spec) {
+  var pages = spec.pages;
+  if (!Array.isArray(pages) || !pages.length) throw fail('pages must list at least one page');
+  if (pages.length > LIMITS.pages) throw fail('At most ' + LIMITS.pages + ' pages');
+  pages.forEach(function(page, p) {
+    var blocks = (page && page.blocks) || [];
+    if (!Array.isArray(blocks)) throw fail('pages[' + p + '].blocks must be a list');
+    if (blocks.length > LIMITS.blocksPerPage) throw fail('At most ' + LIMITS.blocksPerPage + ' blocks on a page');
+    blocks.forEach(function(b, i) {
+      var where = 'pages[' + p + '].blocks[' + i + ']';
+      if (!b || TYPES.indexOf(b.type) === -1) throw fail(where + ': type must be one of ' + TYPES.join(', '));
+      if (![b.x, b.y, b.w, b.h].every(fraction)) throw fail(where + ': x, y, w and h are fractions of the page');
+      if (b.type === 'image') {
+        var src = String(b.src || '');
+        if (!/^data:image\/(png|jpe?g);base64,[A-Za-z0-9+/=]+$/.test(src)) throw fail(where + ': src must be a PNG or JPEG data URL');
+        if (src.length * 0.75 > LIMITS.imageBytes) throw fail(where + ': image is larger than ' + (LIMITS.imageBytes >> 20) + ' MB');
+      }
+      if (b.type === 'table') {
+        if (!Array.isArray(b.columns) || !Array.isArray(b.rows)) throw fail(where + ': a table needs columns and rows');
+        if (b.rows.length > LIMITS.tableRows) b.rows = b.rows.slice(0, LIMITS.tableRows);
+      }
+      if (b.type === 'text' && String(b.text == null ? '' : b.text).length > LIMITS.text) b.text = String(b.text).slice(0, LIMITS.text);
+    });
+  });
+}
+
+module.exports = {
+  name: 'export-pages',
+  description: 'Lay out pages of placed blocks (text, tables, images, shapes) as a PDF — a dashboard as it looked.',
+  requires: [],
+  LIMITS: LIMITS,
+
+  inputSchema: [
+    { name: 'pages', type: 'array', required: true, order: 1,
+      description: '[{ blocks: [{ type: text|table|image|rect|line, x, y, w, h, … }] }], positions as fractions of the page.' },
+    { name: 'title', type: 'string', order: 2 },
+    { name: 'footer', type: 'string', order: 3 },
+    { name: 'landscape', type: 'boolean', order: 4, description: 'Default true.' },
+    { name: 'outputPath', type: 'string', order: 5 }
+  ],
+  outputSchema: [
+    { name: 'filePath', type: 'string' }, { name: 'fileName', type: 'string' },
+    { name: 'contentType', type: 'string' }, { name: 'bytes', type: 'number' }, { name: 'pages', type: 'number' }
+  ],
+
+  execute: async function(ctx) {
+    var input = ctx.input || {};
+    checkFormatRequires('export-pages', 'pdf', pdf);
+    validate(input);
+    var name = (String(input.title || 'dashboard').replace(/[\\/:*?"<>|\u0000-\u001F]+/g, ' ').trim().slice(0, 80) || 'dashboard') + '.pdf';
+    var dir = process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions');
+    fs.mkdirSync(dir, { recursive: true });
+    var filePath = input.outputPath || path.join(fs.mkdtempSync(path.join(dir, 'export-')), name);
+    var done = await pdf.renderPages(filePath, {
+      pages: input.pages, title: input.title, footer: input.footer, landscape: input.landscape !== false
+    });
+    return {
+      filePath: filePath, fileName: path.basename(filePath), contentType: 'application/pdf',
+      bytes: fs.statSync(filePath).size, pages: done.pages
+    };
+  }
+};

package/lib/builtins/export/table.js

@@ -0,0 +1,116 @@
+// export-table — rows to a CSV, Excel or PDF file.
+//
+// The rows arrive already DECIDED: which columns, in what order, under what
+// heading, and for each cell both its raw value and the text a reader saw.
+// This action does not know what a report or a pivot is; whoever calls it
+// worked that out (in BI, report-engine's presentTable). Each format takes
+// what it needs from a row:
+//
+//   csv   the raw values, unformatted — a file for another program
+//   xlsx  the raw values as TYPED cells carrying each column's number format,
+//         so the sheet looks like the screen and still sums
+//   pdf   the display text, exactly as shown
+//
+// Rows come inline (`rows`) or from an NDJSON file (`rowsFile`, one row per
+// line — what streaming/spool writes), so a large result never has to be one
+// array in memory. CSV and Excel stream; a PDF holds its rows (capped, 5,000 by
+// default) because a page is laid out from them.
+
+var fs = require('fs');
+var os = require('os');
+var path = require('path');
+var readline = require('readline');
+var writers = require('../../formats/writers');
+var { checkFormatRequires } = require('../../formats');
+
+function tmpDir() {
+  var dir = process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions');
+  fs.mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+/** A file name from a title: what a person would recognise in their downloads. */
+function fileNameFor(title, extension) {
+  var base = String(title || 'export').replace(/[\\/:*?"<>|\u0000-\u001F]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80) || 'export';
+  return base + '.' + extension;
+}
+
+async function* rowsFrom(input) {
+  if (Array.isArray(input.rows)) { for (var i = 0; i < input.rows.length; i++) yield input.rows[i]; return; }
+  var lines = readline.createInterface({ input: fs.createReadStream(input.rowsFile), crlfDelay: Infinity });
+  for await (var line of lines) if (line.trim()) yield JSON.parse(line);
+}
+
+module.exports = {
+  name: 'export-table',
+  description: 'Write rows to a CSV, Excel (.xlsx) or PDF file. Excel cells stay numbers with their number format; ' +
+               'a PDF shows each cell as it was displayed.',
+  requires: [],
+
+  inputSchema: [
+    { name: 'format', type: 'string', required: true, order: 1, enum: writers.SUPPORTED,
+      description: 'csv, xlsx or pdf.' },
+    { name: 'document', type: 'object', required: true, order: 2,
+      description: '{ title?, subtitle?, footer?, columns: [{ header, kind?: number|date|text, numFmt?, align?, width? }] }. ' +
+                   'numFmt is an Excel number format ("#,##0.00", "0.0%", "mmm yyyy").' },
+    { name: 'rows', type: 'array', order: 3,
+      description: 'Rows: { values: [], display?: [], kind?: detail|subtotal|total }. Or give rowsFile.' },
+    { name: 'rowsFile', type: 'string', order: 4,
+      description: 'An NDJSON file with one row per line, instead of rows.' },
+    { name: 'outputPath', type: 'string', order: 5,
+      description: 'Where to write. Default: a new file in XEPLR_ACTIONS_TMP_DIR named after the title.' },
+    { name: 'maxRows', type: 'number', order: 6,
+      description: 'PDF only: rows to draw before stopping with a note (default 5,000).' }
+  ],
+  outputSchema: [
+    { name: 'filePath', type: 'string' }, { name: 'fileName', type: 'string' },
+    { name: 'contentType', type: 'string' }, { name: 'bytes', type: 'number' },
+    { name: 'rows', type: 'number' }, { name: 'truncated', type: 'boolean' }
+  ],
+
+  execute: async function(ctx) {
+    var input = ctx.input || {};
+    var format = String(input.format || '').toLowerCase();
+    var writer = writers.getWriter(format);
+    checkFormatRequires('export-table', format, writer);
+    var doc = Object.assign({}, input.document || {});
+    if (!Array.isArray(doc.columns) || !doc.columns.length) {
+      var bad = new Error('document.columns must list at least one column');
+      bad.status = 400;
+      throw bad;
+    }
+    if (!Array.isArray(input.rows) && !input.rowsFile) {
+      var none = new Error('Give rows, or a rowsFile');
+      none.status = 400;
+      throw none;
+    }
+    if (format === 'pdf' && input.maxRows) doc.maxRows = input.maxRows;
+
+    var info = writers.INFO[format];
+    var fileName = fileNameFor(doc.title, info.extension);
+    var filePath = input.outputPath || path.join(fs.mkdtempSync(path.join(tmpDir(), 'export-')), fileName);
+    var open = writer.open(filePath, doc);
+    var signal = ctx.signal;
+    try {
+      for await (var row of rowsFrom(input)) {
+        if (signal && signal.aborted) throw new Error('Stopped by request');
+        var more = await open.writeRow(row);
+        // Excel's own limit: the rest are only counted. A PDF keeps counting
+        // too, so its closing note can say how many were left out.
+        if (more === false && format === 'xlsx') { /* keep reading to count */ }
+      }
+      var done = await open.close();
+      return {
+        filePath: filePath,
+        fileName: path.basename(filePath),
+        contentType: info.contentType,
+        bytes: fs.statSync(filePath).size,
+        rows: done.rows,
+        truncated: done.truncated
+      };
+    } catch (err) {
+      try { fs.rmSync(filePath, { force: true }); } catch (e) { /* nothing to clean */ }
+      throw err;
+    }
+  }
+};

package/lib/builtins/file/upload.js

@@ -113,7 +113,12 @@ module.exports = {
         // opaque tenant/app context — see runner.js's ctx.system passthrough.
         metaStore:      ctx.metaStore,
         mtId1: system.mtId1, mtId2: system.mtId2, mtId3: system.mtId3, mtId4: system.mtId4,
-        details:        system.details
+        details:        system.details,
+        // Progress and the stop switch, as db-move passes them: the uploader
+        // supports both, and without them a long import could neither say how
+        // far it had got nor be stopped.
+        onProgress:     system.onProgress || null,
+        signal:         system.signal || null
       });
     } finally {
       queue.stop();

package/lib/builtins/index.js

@@ -61,5 +61,12 @@ module.exports = {
   emailMove:                require('./email/move'),
   emailDelete:              require('./email/delete'),
   emailDownloadEmail:       require('./email/download-email'),
-  emailDownloadAttachments: require('./email/download-attachments')
+  emailDownloadAttachments: require('./email/download-attachments'),
+
+  // Export — rows to CSV / Excel / PDF, and placed blocks to a PDF. The rows
+  // arrive already presented (columns, headings, raw values and display
+  // text); see export/table.js. Peers: fflate (xlsx), pdfkit and
+  // dejavu-fonts-ttf (pdf) — optional, required only by the format used.
+  exportTable:              require('./export/table'),
+  exportPages:              require('./export/pages')
 };

package/lib/drivers/db/duckdb.js

@@ -196,6 +196,9 @@ async function openWithWait(api, key, options, access, timeoutMs) {
  *   config.openTimeoutMs            how long to wait out another holder
  *                                   (default 60s — a load can be slow)
  *   config.maxConnections           default 4
+ *   config.keepIndexes              true to allow indexes in this file. Off by
+ *                                   default — see "a written file keeps no
+ *                                   index" below.
  *
  * READ-ONLY UNLESS ASKED. A caller that forgets `access: 'rw'` gets an error
  * on its first write, in the code doing the writing — not a silent success on
@@ -248,10 +251,78 @@ async function connect(config) {
     max: Math.max(1, parseInt(config.maxConnections || 4, 10))
   };
   entry.pool = { __duck: entry, access: access, file: key };
+  // ':memory:' never outlives its process, so it has no WAL to replay and
+  // nothing for the guard to protect.
+  entry.keepIndexes = Boolean(config.keepIndexes) || key === ':memory:';
+  if (access === 'rw' && key !== ':memory:') {
+    try {
+      await guardOnOpen(entry);
+    } catch (err) {
+      try { instance.closeSync(); } catch (_) {}
+      throw err;
+    }
+  }
   INSTANCE_CACHE[key] = entry;
   return entry.pool;
 }
 
+// ── a written file keeps no index ────────────────────────────────────────
+//
+// DuckDB 1.5.4–1.5.6 (duckdb/duckdb#26106, open upstream): after a process
+// dies with rows still only in the WAL, the next open replays them into the
+// table, and the next checkpoint drops them from every ART index on it. The
+// rows are still there; the index no longer knows them. The first DELETE that
+// reaches one fails with "Failed to delete all rows from index" — FATAL, and
+// the whole file is unusable until the process restarts. Unique or not, any
+// index. With no index there is nothing to fall out of step.
+//
+// So on every read-write open of a file:
+//   1. CHECKPOINT first — folds a replayed WAL into the file before anything
+//      else touches it, the upstream workaround (proven here: kill, reopen,
+//      CHECKPOINT, DELETE succeeds even with an index).
+//   2. DROP every index in it — one left from before this rule, or one that
+//      slipped in, cannot be the thing a later crash breaks.
+// and CREATE INDEX is refused on the pool from then on (see query). Upserts do
+// not need one: they are staged and matched in the move (mergeStaged).
+//
+// A reader cannot trip this: a read-only open replays nothing and deletes
+// nothing, and a file a crash left damaged gets its index dropped the first
+// time anything opens it to write.
+async function guardOnOpen(entry) {
+  var conn = await entry.instance.connect();
+  try {
+    await conn.run('CHECKPOINT');
+    if (entry.keepIndexes) return;
+    var reader = await conn.runAndReadAll(
+      "SELECT schema_name, index_name, table_name FROM duckdb_indexes()"
+    );
+    var found = reader.getRowObjectsJS();
+    for (var i = 0; i < found.length; i++) {
+      await conn.run('DROP INDEX IF EXISTS ' + quoteIdent(found[i].schema_name) + '.' + quoteIdent(found[i].index_name));
+    }
+    if (found.length) {
+      await conn.run('CHECKPOINT');
+      console.warn('[duckdb] ' + entry.key + ': dropped ' + found.length + ' index(es) (' +
+        found.map(function(f) { return f.table_name + '.' + f.index_name; }).join(', ') +
+        ') — a written file keeps none; see duckdb/duckdb#26106');
+    }
+  } finally {
+    try { conn.closeSync(); } catch (_) {}
+  }
+}
+
+var CREATE_INDEX = /^\s*CREATE\s+(UNIQUE\s+)?INDEX\b/i;
+
+function refuseIndex(entry, sql) {
+  if (entry.keepIndexes || !CREATE_INDEX.test(String(sql || ''))) return;
+  throw new Error(
+    'duckdb: CREATE INDEX refused on "' + entry.key + '". A file that is written keeps no index: ' +
+    'DuckDB loses rows from an index after a crash (duckdb/duckdb#26106), and the next DELETE ' +
+    'then fails fatally. Upserts are staged and need none. Open with keepIndexes: true only for ' +
+    'a file that is never written after the index exists.'
+  );
+}
+
 /** Check out a connection, creating one lazily up to `max`, else queueing. */
 async function acquire(entry) {
   if (entry.free.length) return entry.free.pop();
@@ -341,6 +412,7 @@ function plainDeep(v) {
  */
 async function query(pool, sql, params) {
   var entry = entryOf(pool);
+  refuseIndex(entry, sql);
   var conn = await acquire(entry);
   try {
     var api = require('@duckdb/node-api');
@@ -528,7 +600,7 @@ async function ensureMovementColumn(pool, tableName) {
 
 async function ensureUpsertIndex(pool, tableName, primaryKeys) {
   var sql = buildUpsertIndexSql(tableName, primaryKeys);
-  if (sql) await query(pool, sql);
+  if (sql) await query(pool, sql);   // refused on a guarded file — the uploader stages DuckDB upserts instead
 }
 
 // ── introspection ────────────────────────────────────────────────────────
@@ -800,6 +872,11 @@ async function dropStaging(pool, tableName) {
  * under ON CONFLICT DO UPDATE; a row with a NULL key matches nothing, and is
  * kept, as ON CONFLICT kept it.
  *
+ * With replaceFrom { column } it replaces a RANGE: target rows with column >=
+ * the earliest staged value are deleted first ("re-load the last N days" —
+ * the caller reads the source from its cutoff; the boundary here is what that
+ * read actually brought). `replaced` is then the rows deleted.
+ *
  * Returns { inserted, replaced }.
  */
 async function mergeStaged(pool, opts) {
@@ -824,6 +901,21 @@ async function mergeStaged(pool, opts) {
     await conn.run('BEGIN TRANSACTION');
     try {
       var replaced = 0;
+      // A RANGE REPLACED: the target's rows from the EARLIEST STAGED value
+      // on go, and the staged rows take their place — inside this one
+      // transaction, so a crash leaves the old rows, never neither.
+      //
+      // From what was staged, not from a date passed in. The staged values
+      // came through the same reads and conversions as the target's own, so
+      // this boundary is exact; a cutoff computed elsewhere is a time zone
+      // away from it (a DATE read as UTC midnight here, compared at the
+      // source's local midnight there) — and a day deleted but never read
+      // back is a day lost. Nothing staged: MIN is NULL, nothing is deleted.
+      if (opts.replaceFrom) {
+        var col = quoteIdent(opts.replaceFrom.column);
+        var cut = await conn.run('DELETE FROM ' + target + ' WHERE ' + col + ' >= (SELECT MIN(' + col + ') FROM ' + staging + ')');
+        replaced = Number(cut.rowsChanged) || 0;
+      }
       if (pks) {
         var del = await conn.run('DELETE FROM ' + target + ' AS t USING ' + staging + ' AS s WHERE ' +
           pks.map(function(k) { return 't.' + quoteIdent(k) + ' = s.' + quoteIdent(k); }).join(' AND '));
@@ -881,6 +973,9 @@ module.exports = {
 
   ensureMovementColumn: ensureMovementColumn,
   ensureUpsertIndex:    ensureUpsertIndex,
+  // An upsert into DuckDB is always staged: it matches keys in the move, so it
+  // needs no index, which a written DuckDB file must not keep.
+  stagesUpserts:        true,
 
   getTableSchema:  getTableSchema,
   listTables:      listTables,

package/lib/drivers/db/mssql.js

@@ -314,14 +314,15 @@ function toMssqlDatetime(v) {
 
 async function getTableSchema(pool, tableName, schema) {
   var r = await query(pool,
-    'SELECT COLUMN_NAME AS name, DATA_TYPE AS dataType ' +
+    'SELECT COLUMN_NAME AS name, DATA_TYPE AS dataType, CHARACTER_MAXIMUM_LENGTH AS maxLength ' +
     'FROM INFORMATION_SCHEMA.COLUMNS ' +
     'WHERE TABLE_NAME = @p0 AND (@p1 IS NULL OR TABLE_SCHEMA = @p1) ' +
     'ORDER BY ORDINAL_POSITION',
     [tableName, schema || null]
   );
   return r.rows.map(function(row) {
-    return { name: row.name, dataType: row.dataType, udtName: row.dataType };
+    // maxLength -1 is (max): nvarchar(max), varchar(max), varbinary(max).
+    return { name: row.name, dataType: row.dataType, udtName: row.dataType, maxLength: row.maxLength == null ? null : Number(row.maxLength) };
   });
 }
 

package/lib/drivers/db/mysql.js

@@ -311,14 +311,14 @@ function toMysqlDatetime(v) {
 
 async function getTableSchema(pool, tableName, schema) {
   var r = await query(pool,
-    'SELECT COLUMN_NAME AS name, DATA_TYPE AS dataType, COLUMN_TYPE AS udtName ' +
+    'SELECT COLUMN_NAME AS name, DATA_TYPE AS dataType, COLUMN_TYPE AS udtName, CHARACTER_MAXIMUM_LENGTH AS maxLength ' +
     'FROM information_schema.COLUMNS ' +
     'WHERE TABLE_SCHEMA = COALESCE(?, DATABASE()) AND TABLE_NAME = ? ' +
     'ORDER BY ORDINAL_POSITION',
     [schema || null, tableName]
   );
   return r.rows.map(function(row) {
-    return { name: row.name, dataType: row.dataType, udtName: row.udtName };
+    return { name: row.name, dataType: row.dataType, udtName: row.udtName, maxLength: row.maxLength == null ? null : Number(row.maxLength) };
   });
 }
 

package/lib/drivers/db/postgres.js

@@ -290,14 +290,15 @@ async function ensureUpsertIndex(pool, tableName, primaryKeys) {
 // in ordinal-position order. Empty array if the table doesn't exist.
 async function getTableSchema(pool, tableName, schema) {
   var r = await pool.query(
-    "SELECT column_name, data_type, udt_name " +
+    "SELECT column_name, data_type, udt_name, character_maximum_length " +
     "FROM information_schema.columns " +
     "WHERE table_schema = $2 AND table_name = $1 " +
     "ORDER BY ordinal_position",
     [tableName, schema || 'public']
   );
   return r.rows.map(function(row) {
-    return { name: row.column_name, dataType: row.data_type, udtName: row.udt_name };
+    return { name: row.column_name, dataType: row.data_type, udtName: row.udt_name,
+      maxLength: row.character_maximum_length == null ? null : Number(row.character_maximum_length) };
   });
 }
 

package/lib/formats/index.js

@@ -21,8 +21,10 @@ function getFormat(type) {
 function checkFormatRequires(actionName, format, mod) {
   var requires = (mod && mod.requires) || [];
   for (var i = 0; i < requires.length; i++) {
+    // A package of data files (a font) has no main to resolve; its
+    // package.json says it is installed just as well.
     try { require.resolve(requires[i]); }
-    catch (_) {
+    catch (_) { try { require.resolve(requires[i] + '/package.json'); continue; } catch (__) { /* missing — fall through */ }
       throw new Error(
         actionName + ' requires "' + requires[i] + '" for format="' + format + '". ' +
         'Install with: npm install ' + requires[i]