@xeplr/actions 1.1.13 → 1.1.14

package/README.md

@@ -189,6 +189,8 @@ await actions.uploader.upload({
 // → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, durationMs }
 ```
 
+`onProgress({ rowsRead, batches, rowsWritten, phase? })` fires after each batch is read, with `rowsWritten` from the write queue's own count, so a caller can show both and see whether writing keeps up. After the read, while the queue drains, it fires again each time the written count moves (`phase: 'writing'`), never on a bare timer, so a stuck write goes quiet.
+
 Rows are spooled to NDJSON files on disk between source and database. From the first batch it creates, if missing: the target table, `<targetTable>_import_errors` (`movement_id`, `row_num`, `error_description`, `underlying_sql`, `raw_row`, `recorded_at`), a `__xeplr_movement_id__` column, and a unique index on `primaryKeys`. Server dialects also add `__xeplr_id__`; DuckDB does not.
 
 Logical types and what each driver creates:
@@ -310,7 +312,7 @@ node --test test/uploader/reconcile.test.js
 
 | needs nothing | needs a database |
 |---|---|
-| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
+| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
 | `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
 | | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
 | | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |

package/lib/uploader/index.js

@@ -154,9 +154,12 @@ async function upload(opts) {
           // No total, because for a query source there isn't one to know
           // without running it twice. A caller that wants a percentage has to
           // supply its own denominator.
+          //
+          // rowsWritten is what HAS landed, from the write queue's own count,
+          // so a caller can show both and see whether writing keeps up.
           if (opts.onProgress) {
             try {
-              opts.onProgress({ rowsRead: rowsSeen, batches: meta ? meta.batchIndex + 1 : totalBatches });
+              opts.onProgress({ rowsRead: rowsSeen, batches: meta ? meta.batchIndex + 1 : totalBatches, rowsWritten: writtenSoFar(opts) });
             } catch (e) { /* a reporting failure must not fail the movement */ }
           }
         }
@@ -191,7 +194,19 @@ async function upload(opts) {
     }
 
     // Wait for the DB writes to finish (spool has finished feeding the queue).
-    await opts.queue.drain();
+    //
+    // The read is over but the queue can take minutes more to drain, and
+    // nothing reported progress while it did — so a caller watching for
+    // progress saw a busy movement go silent. Reported here whenever the
+    // written count MOVES (never on a bare timer: a stuck write must go quiet).
+    var drainWatch = opts.onProgress ? watchWrites(opts, function() {
+      return { rowsRead: rowsSeen, batches: totalBatches, rowsWritten: writtenSoFar(opts), phase: 'writing' };
+    }) : null;
+    try {
+      await opts.queue.drain();
+    } finally {
+      if (drainWatch) drainWatch.stop();
+    }
 
     var stats = opts.queue.stats(opts.movementId) || {};
     var status = stats.aborted ? 'aborted' : 'completed';
@@ -332,6 +347,31 @@ async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
   return reconciled;
 }
 
+/** Rows this movement has written so far, from the write queue's own count. */
+function writtenSoFar(opts) {
+  try {
+    var st = opts.queue && opts.queue.stats ? opts.queue.stats(opts.movementId) : null;
+    return st && st.completed != null ? st.completed : null;
+  } catch (e) { return null; }
+}
+
+/**
+ * While the queue drains, report progress each time the written count moves.
+ * Checked every `everyMs`; a count that has not moved reports nothing, so a
+ * stuck write goes quiet rather than looking alive.
+ */
+function watchWrites(opts, build, everyMs) {
+  var last = writtenSoFar(opts);
+  var timer = setInterval(function() {
+    var now = writtenSoFar(opts);
+    if (now == null || now === last) return;
+    last = now;
+    try { opts.onProgress(build()); } catch (e) { /* reporting must not fail the movement */ }
+  }, everyMs || 2000);
+  if (timer.unref) timer.unref();
+  return { stop: function() { clearInterval(timer); } };
+}
+
 async function enqueueBatch(opts, rows, columns, batchRef) {
   var driver     = opts.driver;
   var target     = opts.targetTable;
@@ -365,6 +405,7 @@ var { inferColumnType } = require('./infer');
 
 module.exports = {
   upload:          upload,
+  watchWrites:     watchWrites,
   rollback:        rollback,
   inferColumns:    inferColumns,
   inferColumnType: inferColumnType,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.1.13",
+  "version": "1.1.14",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "files": [