@xeplr/actions 1.1.12 → 1.1.14

package/README.md

@@ -189,6 +189,8 @@ await actions.uploader.upload({
 // → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, durationMs }
 ```
 
+`onProgress({ rowsRead, batches, rowsWritten, phase? })` fires after each batch is read, with `rowsWritten` from the write queue's own count, so a caller can show both and see whether writing keeps up. After the read, while the queue drains, it fires again each time the written count moves (`phase: 'writing'`), never on a bare timer, so a stuck write goes quiet.
+
 Rows are spooled to NDJSON files on disk between source and database. From the first batch it creates, if missing: the target table, `<targetTable>_import_errors` (`movement_id`, `row_num`, `error_description`, `underlying_sql`, `raw_row`, `recorded_at`), a `__xeplr_movement_id__` column, and a unique index on `primaryKeys`. Server dialects also add `__xeplr_id__`; DuckDB does not.
 
 Logical types and what each driver creates:
@@ -293,6 +295,8 @@ await driver.query(pool, userSql, params)          // reads only
 | **Rollback is refused for a movement that used `primaryKeys`.** | Upsert stamps the movement id on rows it only updated, so deleting by it would remove rows that already existed. |
 | Movement logs carry identifiers, counts, SQL and the error text — never parameter values or row contents; a failed batch shows one row's primary key, not its data. | Logs are copied into tickets and chats. |
 | DuckDB opens **read-only unless the call says `access: 'rw'`**; no environment variable can grant write. A file already open read-only in the process refuses a write. Opening waits on the lock (`openTimeoutMs`, default 60 s). | One read-write process locks every other process out, even readers. Write access belongs at the call site that writes. |
+| **Every DuckDB value comes out plain**, from `query` and `fetchStream` alike: a number, text, boolean, `Date`, array or object, never a BigInt or a DuckDB wrapper object. DECIMAL is a number; DATE and TIMESTAMP are `Date`s; a whole number too big to hold exactly (beyond ±9,007,199,254,740,991) is **text**, never rounded; lists, structs and maps are arrays and objects; TIME, INTERVAL, UUID and any other type are text. `fetchStream` has DuckDB do the conversion inside its engine by wrapping the query (about +3% on a million rows); a query with nothing to convert runs as given. | A streamed read used to hand DECIMAL and TIMESTAMP back as wrapper objects with a BigInt inside, and the first `JSON.stringify` downstream failed with "Do not know how to serialize a BigInt". Text for huge numbers is what the Postgres, MySQL and SQL Server drivers already return. |
+| **A DuckDB timestamp without a zone is read as UTC**; one with a zone keeps its exact instant. | Data is kept in UTC; the viewer's time zone is applied when rendering, never in the driver. |
 | `meta-store-knex` requires `service` and `applicationId`, and scopes every statement by `applicationId`. | `import_meta` is shared by every app; unattributed rows, or a rollback decided from another app's row, are worse than refusing. |
 | Mail filenames are sanitised (`[^\w.-]` → `_`, all-dot names → `attachment`, max 180 chars) and prefixed with a random id. | Attachment names come from the sender and could escape the output directory; common names collide. |
 | The system mail sender refuses `text`, `bcc`, `from`, `replyTo` rather than dropping them. | A silently lost bcc is a compliance problem found a year later. |
@@ -308,8 +312,8 @@ node --test test/uploader/reconcile.test.js
 
 | needs nothing | needs a database |
 |---|---|
-| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
-| `test/drivers/duckdb.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
+| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
+| `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
 | | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
 | | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |
 

package/lib/drivers/db/duckdb.js

@@ -299,11 +299,27 @@ async function close(pool) {
 
 // BigInt is what DuckDB hands back for BIGINT/HUGEINT, and it poisons anything
 // downstream that does arithmetic or JSON.stringify on a result. Narrowed to a
-// number where that is lossless, left alone where it is not — silently losing
-// precision on a genuinely huge value would be worse than an awkward type.
+// number where that is lossless, and to TEXT where it is not: silently losing
+// digits on a genuinely huge value would be wrong, and a BigInt left in a row
+// crashes the first JSON.stringify that meets it. Text is also what Postgres,
+// MySQL and SQL Server already return for such a value, so every source
+// behaves alike.
 function unBig(v) {
   if (typeof v !== 'bigint') return v;
-  return (v <= 9007199254740991n && v >= -9007199254740991n) ? Number(v) : v;
+  return (v <= 9007199254740991n && v >= -9007199254740991n) ? Number(v) : v.toString();
+}
+
+// The same, all the way down: a BigInt can sit INSIDE a value the normal read
+// returns (an INTERVAL's micros, a list of BIGINTs). Arrays and plain objects
+// keep their shape; a Date, a string or a number is left as it is.
+function plainDeep(v) {
+  if (typeof v === 'bigint') return unBig(v);
+  if (v === null || typeof v !== 'object' || v instanceof Date) return v;
+  if (Array.isArray(v)) return v.map(plainDeep);
+  if (Object.getPrototypeOf(v) !== Object.prototype) return v;
+  var out = {};
+  for (var k in v) out[k] = plainDeep(v[k]);
+  return out;
 }
 
 /**
@@ -337,7 +353,7 @@ async function query(pool, sql, params) {
     var columns = reader.columnNames();
     var rows = reader.getRowObjectsJS().map(function(row) {
       var out = {};
-      for (var k in row) out[k] = unBig(row[k]);
+      for (var k in row) out[k] = plainDeep(row[k]);
       return out;
     });
     return { rows: rows, rowCount: rows.length, columns: columns };
@@ -551,6 +567,108 @@ async function listProcedures(_pool, _schema) {
   return [];
 }
 
+// ── plain values out of a stream ─────────────────────────────────────────
+//
+// A streamed chunk (getRowObjects) hands DECIMAL, TIMESTAMP, INTERVAL, UUID
+// and the nested types back as DuckDB WRAPPER objects, several with a BigInt
+// inside. They are not the values the rest of the code expects, and the first
+// JSON.stringify that meets one fails ("Do not know how to serialize a
+// BigInt") — which is how a report grouped by an amount or a timestamp from a
+// cube used to die.
+//
+// Converting each wrapper in JavaScript doubles the cost of a streamed read.
+// So DuckDB converts them inside its own engine instead, by wrapping the
+// query: measured on 1M rows, +2% (+6% with the timestamp turned into a Date
+// here), against +100% for the JS converter.
+//
+//   passes through   booleans, integers (BigInt narrowed by unBig), floats,
+//                    text, enums
+//   DECIMAL          DOUBLE: what the non-streamed read (getRowObjectsJS)
+//                    returns too
+//   DATE, TIMESTAMP* epoch milliseconds, turned into a Date as the row is
+//                    built — the same type Postgres and the normal read give.
+//                    A timestamp WITHOUT a zone is read as UTC (data is kept
+//                    in UTC; the viewer's zone is applied when rendering). One
+//                    WITH a zone keeps its exact instant: epoch_ms is taken on
+//                    it directly, never after a cast that would apply the
+//                    session's zone.
+//   LIST, STRUCT,    JSON, parsed back here: arrays and objects, the shape the
+//   MAP, ARRAY       normal read gives
+//   anything else    VARCHAR (TIME, INTERVAL, UUID, …). A type nobody has met
+//                    yet becomes text rather than a crash.
+//
+// When no column needs converting, the query runs exactly as given.
+var PASS_THROUGH_TYPES = null;
+var DATE_TYPES = null;
+var NESTED_TYPES = null;
+function typeSets() {
+  if (PASS_THROUGH_TYPES) return;
+  var T = require('@duckdb/node-api').DuckDBTypeId;
+  PASS_THROUGH_TYPES = new Set([T.BOOLEAN, T.TINYINT, T.SMALLINT, T.INTEGER, T.BIGINT, T.HUGEINT,
+    T.UTINYINT, T.USMALLINT, T.UINTEGER, T.UBIGINT, T.UHUGEINT, T.FLOAT, T.DOUBLE,
+    T.VARCHAR, T.ENUM, T.SQLNULL].filter(function(x) { return x !== undefined; }));
+  DATE_TYPES = {};
+  DATE_TYPES[T.DATE] = 'plain';
+  DATE_TYPES[T.TIMESTAMP] = 'plain';
+  DATE_TYPES[T.TIMESTAMP_S] = 'plain';
+  DATE_TYPES[T.TIMESTAMP_MS] = 'plain';
+  DATE_TYPES[T.TIMESTAMP_NS] = 'plain';
+  DATE_TYPES[T.TIMESTAMP_TZ] = 'zoned';
+  NESTED_TYPES = new Set([T.LIST, T.STRUCT, T.MAP, T.ARRAY, T.UNION].filter(function(x) { return x !== undefined; }));
+}
+
+function stripTrailingSemicolon(sql) {
+  return String(sql).replace(/[\s;]+$/, '');
+}
+
+async function plainValuesPlan(conn, sql) {
+  typeSets();
+  var prepared = await conn.prepare(sql);
+  var count = prepared.columnCount;
+  var select = [];
+  var inner = [];
+  var dateColumns = [];
+  var jsonColumns = [];
+  var changed = false;
+  for (var i = 0; i < count; i++) {
+    var name = prepared.columnName(i);
+    var typeId = prepared.columnTypeId(i);
+    var col = '"__c' + i + '"';
+    inner.push(col);
+    var expr;
+    if (PASS_THROUGH_TYPES.has(typeId)) {
+      expr = col;
+    } else if (DATE_TYPES[typeId] === 'zoned') {
+      expr = 'epoch_ms(' + col + ')';
+      dateColumns.push(name);
+      changed = true;
+    } else if (DATE_TYPES[typeId] === 'plain') {
+      expr = 'epoch_ms(CAST(' + col + ' AS TIMESTAMP))';
+      dateColumns.push(name);
+      changed = true;
+    } else if (typeId === require('@duckdb/node-api').DuckDBTypeId.DECIMAL) {
+      expr = 'CAST(' + col + ' AS DOUBLE)';
+      changed = true;
+    } else if (NESTED_TYPES.has(typeId)) {
+      expr = 'CAST(to_json(' + col + ') AS VARCHAR)';
+      jsonColumns.push(name);
+      changed = true;
+    } else {
+      expr = 'CAST(' + col + ' AS VARCHAR)';
+      changed = true;
+    }
+    select.push(expr + ' AS ' + quoteIdent(name));
+  }
+  if (!changed) return { sql: sql, dateColumns: [], jsonColumns: [] };
+  // Positional renames (__c0, __c1, …) so two source columns with one name
+  // stay two columns, whatever they are called.
+  return {
+    sql: 'SELECT ' + select.join(', ') + ' FROM (' + stripTrailingSemicolon(sql) + '\n) AS "__plain"(' + inner.join(', ') + ')',
+    dateColumns: dateColumns,
+    jsonColumns: jsonColumns
+  };
+}
+
 // ── streaming read ───────────────────────────────────────────────────────
 
 /**
@@ -565,8 +683,11 @@ async function* fetchStream(pool, opts) {
   var entry = entryOf(pool);
   var conn = await acquire(entry);
   try {
-    var result = await conn.stream(opts.sql, (opts.params && opts.params.length) ? opts.params : undefined);
+    var plan = await plainValuesPlan(conn, opts.sql);
+    var result = await conn.stream(plan.sql, (opts.params && opts.params.length) ? opts.params : undefined);
     var names = result.columnNames();
+    var dates = plan.dateColumns;
+    var jsons = plan.jsonColumns;
     while (true) {
       var chunk = await result.fetchChunk();
       if (!chunk || chunk.rowCount === 0) break;
@@ -575,6 +696,14 @@ async function* fetchStream(pool, opts) {
         var row = rows[i];
         var out = {};
         for (var k in row) out[k] = unBig(row[k]);
+        for (var d = 0; d < dates.length; d++) {
+          var v = out[dates[d]];
+          if (v !== null && v !== undefined) out[dates[d]] = new Date(v);
+        }
+        for (var j = 0; j < jsons.length; j++) {
+          var jv = out[jsons[j]];
+          if (typeof jv === 'string') out[jsons[j]] = JSON.parse(jv);
+        }
         yield out;
       }
     }

package/lib/uploader/index.js

@@ -154,9 +154,12 @@ async function upload(opts) {
           // No total, because for a query source there isn't one to know
           // without running it twice. A caller that wants a percentage has to
           // supply its own denominator.
+          //
+          // rowsWritten is what HAS landed, from the write queue's own count,
+          // so a caller can show both and see whether writing keeps up.
           if (opts.onProgress) {
             try {
-              opts.onProgress({ rowsRead: rowsSeen, batches: meta ? meta.batchIndex + 1 : totalBatches });
+              opts.onProgress({ rowsRead: rowsSeen, batches: meta ? meta.batchIndex + 1 : totalBatches, rowsWritten: writtenSoFar(opts) });
             } catch (e) { /* a reporting failure must not fail the movement */ }
           }
         }
@@ -191,7 +194,19 @@ async function upload(opts) {
     }
 
     // Wait for the DB writes to finish (spool has finished feeding the queue).
-    await opts.queue.drain();
+    //
+    // The read is over but the queue can take minutes more to drain, and
+    // nothing reported progress while it did — so a caller watching for
+    // progress saw a busy movement go silent. Reported here whenever the
+    // written count MOVES (never on a bare timer: a stuck write must go quiet).
+    var drainWatch = opts.onProgress ? watchWrites(opts, function() {
+      return { rowsRead: rowsSeen, batches: totalBatches, rowsWritten: writtenSoFar(opts), phase: 'writing' };
+    }) : null;
+    try {
+      await opts.queue.drain();
+    } finally {
+      if (drainWatch) drainWatch.stop();
+    }
 
     var stats = opts.queue.stats(opts.movementId) || {};
     var status = stats.aborted ? 'aborted' : 'completed';
@@ -332,6 +347,31 @@ async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
   return reconciled;
 }
 
+/** Rows this movement has written so far, from the write queue's own count. */
+function writtenSoFar(opts) {
+  try {
+    var st = opts.queue && opts.queue.stats ? opts.queue.stats(opts.movementId) : null;
+    return st && st.completed != null ? st.completed : null;
+  } catch (e) { return null; }
+}
+
+/**
+ * While the queue drains, report progress each time the written count moves.
+ * Checked every `everyMs`; a count that has not moved reports nothing, so a
+ * stuck write goes quiet rather than looking alive.
+ */
+function watchWrites(opts, build, everyMs) {
+  var last = writtenSoFar(opts);
+  var timer = setInterval(function() {
+    var now = writtenSoFar(opts);
+    if (now == null || now === last) return;
+    last = now;
+    try { opts.onProgress(build()); } catch (e) { /* reporting must not fail the movement */ }
+  }, everyMs || 2000);
+  if (timer.unref) timer.unref();
+  return { stop: function() { clearInterval(timer); } };
+}
+
 async function enqueueBatch(opts, rows, columns, batchRef) {
   var driver     = opts.driver;
   var target     = opts.targetTable;
@@ -365,6 +405,7 @@ var { inferColumnType } = require('./infer');
 
 module.exports = {
   upload:          upload,
+  watchWrites:     watchWrites,
   rollback:        rollback,
   inferColumns:    inferColumns,
   inferColumnType: inferColumnType,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.1.12",
+  "version": "1.1.14",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "files": [