@xeplr/actions 1.1.21 → 1.1.22

package/README.md

@@ -283,7 +283,7 @@ await actions.uploader.upload({
   signal,                 // optional AbortSignal: stop on request
   staged                  // optional: see Staged loads
 })
-// → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, merged, durationMs }
+// → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, errorRows, aborted, merged, durationMs }
 ```
 
 `onProgress({ rowsRead, batches, rowsWritten, phase? })` fires after each batch is read, with `rowsWritten` from the write queue's own count, so a caller can show both and see whether writing keeps up. After the read, while the queue drains, it fires again each time the written count moves (`phase: 'writing'`), never on a bare timer, so a stuck write goes quiet.
@@ -292,6 +292,8 @@ await actions.uploader.upload({
 
 Rows are spooled to NDJSON files on disk between source and database. From the first batch it creates, if missing: the target table, `<targetTable>_import_errors` (`movement_id`, `row_num`, `error_description`, `underlying_sql`, `raw_row`, `recorded_at`), a `__xeplr_movement_id__` column, and a unique index on `primaryKeys` (not on DuckDB: its upserts are staged and keep none). Server dialects also add `__xeplr_id__`; DuckDB does not.
 
+**Rows the database refuses are kept, in `<targetTable>_import_errors`.** When a batch fails, the queue splits it until it finds the rows the database will not take; each one is written to the error table with `row_num` (its 1-based place in the source, so the first data row is 1), `error_description` (the database's own message and code, e.g. `… violates check constraint "t_qty_check" [23514]`), the statement, and `raw_row` (the row as JSON, every column). They load nowhere else, so this table is the only way to see, fix and re-load them. `errorRows` in the result says how many were recorded: `{ recorded, notRecorded, reason? }`. Recording is best-effort and never fails the movement; `notRecorded` counts rows that could not be written (with the first `reason`). Rows dropped because the movement gave up (stopped, aborted) are counted in `dropped` but were never refused, so they are not recorded. Rolling a movement back deletes its error rows too. A queue's own `onErrorTable` hook still runs first. The statement is built once for every dialect (`lib/drivers/db/errorRows.js`), exposed per driver as `buildInsertErrorRowsSql(table, movementId, entries)`.
+
 Logical types and what each driver creates:
 
 | type | postgres | mysql | mssql | duckdb |
@@ -454,7 +456,7 @@ node --test test/uploader/reconcile.test.js
 
 | needs nothing | needs a database |
 |---|---|
-| `test/formats/pasted-rows.test.js` (its last case loads into Postgres when `PG_HOST`/`PG_PORT`/`PG_USER`/`PG_PASSWORD`/`PG_DATABASE` are set, else skipped), `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
+| `test/formats/pasted-rows.test.js` (its last case loads into Postgres when `PG_HOST`/`PG_PORT`/`PG_USER`/`PG_PASSWORD`/`PG_DATABASE` are set, else skipped), `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, which needs `@xeplr/db`: a peer dependency, installed for development) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js` (also honours `PG_HOST`/`PG_PORT`/`PG_USER`), `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
 | `test/list-columns-length.test.js` (`XEPLR_TEST_PG_URL`, `MYSQL_TEST_URL`, `MSSQL_TEST_URL`; each skipped when unset), `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js`, `test/duckdb-index-guard.test.js`, `test/staged-load.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
 | | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
 | | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |

package/lib/drivers/db/duckdb.js

@@ -966,6 +966,10 @@ function buildRetypeToLocalDateTimeSql(tableName, column) {
   return 'ALTER TABLE ' + quoteIdent(tableName) + ' ALTER COLUMN ' + quoteIdent(column) + ' SET DATA TYPE TIMESTAMP';
 }
 
+// Rows the database refused, into <table>_import_errors — one builder for every dialect.
+var errorRows = require('./errorRows');
+var placeholders = require('./procedure').placeholders;
+
 module.exports = {
   buildRetypeToLocalDateTimeSql: buildRetypeToLocalDateTimeSql,
   requires: ['@duckdb/node-api'],
@@ -979,6 +983,9 @@ module.exports = {
   quoteIdent:               quoteIdent,
   buildCreateTableSql:      buildCreateTableSql,
   buildCreateErrorTableSql: buildCreateErrorTableSql,
+  buildInsertErrorRowsSql: function(tableName, movementId, entries) {
+    return errorRows.buildInsertErrorRowsSql(quoteIdent, placeholders('postgres'), tableName, movementId, entries);
+  },
   buildUpsertIndexSql:      buildUpsertIndexSql,
   buildAlterTableAddSql:    buildAlterTableAddSql,
   buildInsertSql:           buildInsertSql,

package/lib/drivers/db/errorRows.js

@@ -0,0 +1,37 @@
+// ROWS THE DATABASE REFUSED, written to `<table>_import_errors`.
+//
+// Every dialect creates that table (buildCreateErrorTableSql) with the same
+// columns; only the quoting and the bind marks differ. So the statement is
+// built once, here, and each driver passes its own quoteIdent and mark.
+
+var COLUMNS = ['movement_id', 'row_num', 'error_description', 'underlying_sql', 'raw_row'];
+
+/**
+ * One INSERT for several refused rows.
+ *
+ * @param {function} quoteIdent  the driver's identifier quoting
+ * @param {function} mark        bind mark for param n (0-based): '?', '$1', '@p0'
+ * @param {string}   tableName   the TARGET table — the error table is named from it
+ * @param {string}   movementId
+ * @param {Array<{rowNum, error, sql, row}>} entries
+ * @returns {{ sql: string, params: any[] }}
+ */
+function buildInsertErrorRowsSql(quoteIdent, mark, tableName, movementId, entries) {
+  var params = [];
+  var groups = entries.map(function(e) {
+    var marks = [];
+    [movementId, e.rowNum == null ? null : e.rowNum, e.error || null, e.sql || null,
+      e.row == null ? null : JSON.stringify(e.row)].forEach(function(v) {
+      marks.push(mark(params.length));
+      params.push(v);
+    });
+    return '(' + marks.join(', ') + ')';
+  });
+  return {
+    sql: 'INSERT INTO ' + quoteIdent(tableName + '_import_errors') +
+      ' (' + COLUMNS.map(quoteIdent).join(', ') + ') VALUES ' + groups.join(', '),
+    params: params
+  };
+}
+
+module.exports = { buildInsertErrorRowsSql: buildInsertErrorRowsSql };

package/lib/drivers/db/mssql.js

@@ -450,6 +450,10 @@ async function rollbackMovement(pool, tableName, movementId) {
   return result;
 }
 
+// Rows the database refused, into <table>_import_errors — one builder for every dialect.
+var errorRows = require('./errorRows');
+var placeholders = require('./procedure').placeholders;
+
 module.exports = {
   requires: ['mssql'],
   maxParams: MAX_PARAMS,
@@ -465,6 +469,9 @@ module.exports = {
   quoteIdent:               quoteIdent,
   buildCreateTableSql:      buildCreateTableSql,
   buildCreateErrorTableSql: buildCreateErrorTableSql,
+  buildInsertErrorRowsSql: function(tableName, movementId, entries) {
+    return errorRows.buildInsertErrorRowsSql(quoteIdent, placeholders('mssql'), tableName, movementId, entries);
+  },
   buildAlterTableAddSql:    buildAlterTableAddSql,
   buildInsertSql:           buildInsertSql,
 

package/lib/drivers/db/mysql.js

@@ -451,6 +451,10 @@ async function rollbackMovement(pool, tableName, movementId) {
   return result;
 }
 
+// Rows the database refused, into <table>_import_errors — one builder for every dialect.
+var errorRows = require('./errorRows');
+var placeholders = require('./procedure').placeholders;
+
 module.exports = {
   requires: ['mysql2'],
   maxParams: MAX_PARAMS,
@@ -466,6 +470,9 @@ module.exports = {
   quoteIdent:               quoteIdent,
   buildCreateTableSql:      buildCreateTableSql,
   buildCreateErrorTableSql: buildCreateErrorTableSql,
+  buildInsertErrorRowsSql: function(tableName, movementId, entries) {
+    return errorRows.buildInsertErrorRowsSql(quoteIdent, placeholders('mysql'), tableName, movementId, entries);
+  },
   buildAlterTableAddSql:    buildAlterTableAddSql,
   buildInsertSql:           buildInsertSql,
 

package/lib/drivers/db/postgres.js

@@ -403,6 +403,10 @@ function buildRetypeToLocalDateTimeSql(tableName, column) {
   return 'ALTER TABLE ' + quoteIdent(tableName) + ' ALTER COLUMN ' + quoteIdent(column) + ' TYPE timestamp USING ' + quoteIdent(column) + " AT TIME ZONE 'UTC'";
 }
 
+// Rows the database refused, into <table>_import_errors — one builder for every dialect.
+var errorRows = require('./errorRows');
+var placeholders = require('./procedure').placeholders;
+
 module.exports = {
   buildRetypeToLocalDateTimeSql: buildRetypeToLocalDateTimeSql,
   requires: ['pg'],
@@ -418,6 +422,9 @@ module.exports = {
   quoteIdent:              quoteIdent,
   buildCreateTableSql:     buildCreateTableSql,
   buildCreateErrorTableSql:buildCreateErrorTableSql,
+  buildInsertErrorRowsSql: function(tableName, movementId, entries) {
+    return errorRows.buildInsertErrorRowsSql(quoteIdent, placeholders('postgres'), tableName, movementId, entries);
+  },
   buildUpsertIndexSql:     buildUpsertIndexSql,
   buildAlterTableAddSql:   buildAlterTableAddSql,
   buildInsertSql:          buildInsertSql,

package/lib/uploader/index.js

@@ -96,6 +96,17 @@ async function upload(opts) {
   var onStop = opts.signal ? function() { opts.queue.abort(opts.movementId, 'stopped'); } : null;
   if (onStop) opts.signal.addEventListener('abort', onStop, { once: true });
 
+  // WHERE A REFUSED ROW GOES: `<targetTable>_import_errors`, with the
+  // database's reason. The queue splits a failing batch down to the rows the
+  // database refuses and hands each one to its onErrorTable hook — whose
+  // default does nothing, so every one of them used to vanish, leaving only a
+  // count ("63 dropped") and nothing anyone could look at, fix or re-load.
+  var refused = recordRefusedRows(opts, driver, pool);
+  // Each row's place in the source (1-based), to say WHICH row was refused.
+  // Keyed by the row object itself: a split batch slices the same objects, so
+  // the number survives however far the queue bisects.
+  var rowNumbers = refused.rowNumbers;
+
   try {
     var summary = await spool(opts.source, {
       runId:        opts.movementId,
@@ -156,6 +167,9 @@ async function upload(opts) {
         onBatch: async function(rows, meta) {
           if (opts.signal && opts.signal.aborted) throw stoppedError();
           if (!columns) throw new Error('uploader: onBatch before beforeAll — spool contract violated');
+          for (var n = 0; n < rows.length; n++) {
+            if (rows[n] && typeof rows[n] === 'object') rowNumbers.set(rows[n], rowsSeen + n + 1);
+          }
           for (var off = 0; off < rows.length; off += effectiveBatchSize) {
             var chunk = rows.slice(off, off + effectiveBatchSize);
             // A NAME FOR THIS BATCH, carried all the way to the failure.
@@ -231,6 +245,7 @@ async function upload(opts) {
     } finally {
       if (drainWatch) drainWatch.stop();
     }
+    var errorRows = await refused.flush();
     // Stopped while the queue drained: not a finished movement.
     if (opts.signal && opts.signal.aborted) throw stoppedError();
 
@@ -269,12 +284,17 @@ async function upload(opts) {
       totalBatches: totalBatches,
       completed:    stats.completed || 0,
       dropped:      stats.dropped || 0,
+      // Of the dropped rows, how many are in the error table to look at —
+      // { recorded, notRecorded, reason? }. Rows dropped without the database
+      // refusing them (a movement that gave up) are counted, never recorded.
+      errorRows:    errorRows,
       aborted:      !!stats.aborted,
       // Staged only: what the move put in the target — { inserted, replaced }.
       merged:       merged,
       durationMs:   Date.now() - startedAt
     };
   } finally {
+    refused.detach();
     // A staged load that did not move its rows: stopped, failed, aborted.
     // Its staging table is all it wrote, and dropping it is the whole undo.
     if (staged && !merged && pool) {
@@ -290,6 +310,66 @@ async function upload(opts) {
   }
 }
 
+// How many refused rows to write in one INSERT into the error table.
+var ERROR_ROWS_PER_INSERT = 200;
+
+/**
+ * Hook this movement into the queue's onErrorTable, writing every row the
+ * database refuses to `<targetTable>_import_errors` — row number, reason,
+ * the statement, and the row itself as JSON. Whatever hook the caller gave
+ * the queue still runs first. Writing is best-effort: a failure to record is
+ * counted and reported, never a failure of the movement.
+ *
+ * @returns {{ rowNumbers: WeakMap, flush: () => Promise<{recorded, notRecorded, reason?}>, detach: () => void }}
+ */
+function recordRefusedRows(opts, driver, pool) {
+  var queue = opts.queue;
+  var previous = queue.onErrorTable;
+  var rowNumbers = new WeakMap();
+  var pending = [];
+  var writes = [];
+  var tally = { recorded: 0, notRecorded: 0, reason: null };
+  var canWrite = typeof driver.buildInsertErrorRowsSql === 'function';
+
+  function write(entries) {
+    if (!canWrite) {
+      tally.notRecorded += entries.length;
+      tally.reason = tally.reason || 'this database driver cannot write the error table';
+      return;
+    }
+    var built = driver.buildInsertErrorRowsSql(opts.targetTable, opts.movementId, entries);
+    writes.push(driver.query(pool, built.sql, built.params).then(
+      function() { tally.recorded += entries.length; },
+      function(err) { tally.notRecorded += entries.length; tally.reason = tally.reason || err.message; }
+    ));
+  }
+
+  queue.onErrorTable = function(info) {
+    try { if (previous) previous.apply(queue, arguments); } catch (_) { /* the caller's hook cannot stop recording */ }
+    var item = info && info.item;
+    if (!item || item.movementId !== opts.movementId) return;
+    var rows = (item.meta && item.meta.rows) || [];
+    var reason = (info.error && info.error.message) || info.reason || 'unknown error';
+    if (info.errorCode != null) reason = reason + ' [' + info.errorCode + ']';
+    rows.forEach(function(row) {
+      pending.push({ rowNum: rowNumbers.get(row), error: reason, sql: rows.length === 1 ? item.sql : null, row: row });
+    });
+    while (pending.length >= ERROR_ROWS_PER_INSERT) write(pending.splice(0, ERROR_ROWS_PER_INSERT));
+  };
+
+  return {
+    rowNumbers: rowNumbers,
+    flush: async function() {
+      if (pending.length) write(pending.splice(0));
+      await Promise.all(writes);
+      var out = { recorded: tally.recorded, notRecorded: tally.notRecorded };
+      if (tally.reason) out.reason = tally.reason;
+      return out;
+    },
+    detach: function() { queue.onErrorTable = previous; }
+  };
+}
+
 /** The error a stopped movement ends with: code 'STOPPED', so a caller can tell it from a failure. */
 function stoppedError() {
   var err = new Error('Stopped by request');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.1.21",
+  "version": "1.1.22",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "files": [
@@ -37,6 +37,7 @@
   },
   "peerDependencies": {
     "@duckdb/node-api": "1.5.5-r.3",
+    "@xeplr/db": "^1.0.8",
     "@xeplr/email": "^1.0.0",
     "@xeplr/utils": "^1.0.0",
     "csv-parse": "^7.0.2",
@@ -117,6 +118,7 @@
   },
   "devDependencies": {
     "@duckdb/node-api": "^1.5.5-r.3",
+    "@xeplr/db": "^1.0.8",
     "@xeplr/utils": "^1.0.0",
     "csv-parse": "^7.0.2",
     "dejavu-fonts-ttf": "^2.37.3",