@xeplr/actions 1.1.1 → 1.1.6

package/lib/runner.js

@@ -26,7 +26,7 @@ async function runAction(first, second, third) {
 
   var input;
   try {
-    input = applySchema(def.inputSchema, opts.input || {}, 'input');
+    input = applySchema(def.inputSchema, coerceScalars(def.inputSchema, opts.input || {}), 'input');
   } catch (err) {
     return finish(start, 'failed', null, err);
   }
@@ -43,6 +43,51 @@ async function runAction(first, second, third) {
   }
 }
 
+/**
+ * ONE VALUE WHERE A LIST WAS DECLARED MEANS A LIST OF ONE.
+ *
+ * Without this, an `array` field can only ever be given a literal array — and
+ * that quietly breaks the thing every caller of this package is built to do,
+ * which is BIND A REFERENCE. A workflow step writes `to = {params.email}`; the
+ * engine interpolates that to a string; applySchema then rejects it for not
+ * being an array, and the run fails on a step that reads as obviously correct.
+ * Every array-typed input was unbindable, in every action, for every consumer.
+ *
+ * It also makes email-send's own promise true. Its schema has said "a bare
+ * string is accepted and wrapped" since it was written, and nothing wrapped
+ * anything — applySchema refused it before execute() ever ran.
+ *
+ * Deliberately NARROW:
+ *
+ *   - only for fields the schema declares `array`, so nothing else can be
+ *     reshaped by accident;
+ *   - only when the value is a scalar. An object stays an object and fails
+ *     the type check as it should, because `{a:1}` is a mistake, not a
+ *     one-element list;
+ *   - null and undefined are left alone, so `required` still means required
+ *     and a default still fills in;
+ *   - '' is left alone, so an empty box does not become `['']` — a list
+ *     containing one empty string is a value, and "unset" is what was meant.
+ *
+ * No JSON parsing here. A string that looks like JSON is the CALLER's to
+ * parse: this layer cannot tell `"[1,2]"` the literal text from `[1,2]` the
+ * list, and guessing would make a value's meaning depend on its punctuation.
+ */
+function coerceScalars(schema, input) {
+  if (!Array.isArray(schema)) return input;
+  var out = null;
+  for (var i = 0; i < schema.length; i++) {
+    var field = schema[i];
+    if (!field || field.type !== 'array') continue;
+    var val = input[field.name];
+    if (val === undefined || val === null || val === '') continue;
+    if (Array.isArray(val) || typeof val === 'object') continue;
+    if (!out) out = Object.assign({}, input);
+    out[field.name] = [val];
+  }
+  return out || input;
+}
+
 // Accept either the options-object form or the shorthand (action, input, opts).
 function normalizeArgs(first, second, third) {
   // Shorthand form: first arg is a string name or an action module.

package/lib/streaming/spool.js

@@ -17,7 +17,15 @@ var DEFAULTS = {
   batchSize:     5000,
   maxInFlight:   1,
   ratePerSecond: null,
-  maxMemoryMB:   256
+  maxMemoryMB:   256,
+  // How many batches may be buffered BEYOND the ones being processed. The
+  // reader blocks past this, which pauses the source when the source can be
+  // paused. 1 keeps one batch filling while another is written — the overlap
+  // this design is for — without letting a fast source outrun a slow target.
+  //
+  // Raising it trades memory for tolerance of a bursty target: peak footprint
+  // is roughly (maxInFlight + maxReadAhead) x batchSize rows.
+  maxReadAhead:  1
 };
 
 /**
@@ -55,6 +63,30 @@ async function spool(source, opts) {
   var rl  = rateLimiter(opts.ratePerSecond);
   var mem = memoryMonitor(opts.maxMemoryMB);
 
+  // The reader's own governor. Counts batches scheduled but not yet finished;
+  // wait() resolves once that falls back within maxReadAhead. Separate from
+  // `sem`, which bounds how many run AT ONCE — this bounds how far the reader
+  // may get ahead of them.
+  var readAhead = (function(limit) {
+    var outstanding = 0;
+    var waiters = [];
+    return {
+      enter: function() { outstanding++; },
+      leave: function() {
+        outstanding = Math.max(0, outstanding - 1);
+        // Wake everyone and let them re-test: there is only ever one reader,
+        // so this is at most one resumption.
+        var w = waiters; waiters = [];
+        w.forEach(function(resolve) { resolve(); });
+      },
+      wait: async function() {
+        while (outstanding > limit) {
+          await new Promise(function(resolve) { waiters.push(resolve); });
+        }
+      }
+    };
+  })(Math.max(0, opts.maxReadAhead));
+
   // Ordered checkpoint tracking: batches may complete out of order when
   // maxInFlight > 1. `lastContiguous` = highest N where every batch 0..N has
   // reported done. Only checkpoint that highwater — safer for resume.
@@ -108,8 +140,9 @@ async function spool(source, opts) {
   // Adds to the pendingChain to preserve arrival order into the semaphore,
   // then the semaphore itself gates concurrency (maxInFlight).
   function schedule(cf, batchIndex, isLast) {
+    readAhead.enter();
     pendingChain = pendingChain.then(async function() {
-      if (firstError) return;              // short-circuit on failure
+      if (firstError) { readAhead.leave(); return; }   // short-circuit on failure
 
       var release = await sem.acquire();
 
@@ -157,6 +190,10 @@ async function spool(source, opts) {
       } catch (err) {
         if (!firstError) firstError = err;
       } finally {
+        // Before release(), and unconditionally — a batch that threw still
+        // stops being outstanding, or a reader waiting on room would block on
+        // a batch that is never coming back.
+        readAhead.leave();
         release();
       }
     });
@@ -186,6 +223,29 @@ async function spool(source, opts) {
         schedule(cf, idx, false);
         fileIndex++;
         await openNextFile();
+
+        // WAIT FOR ROOM before filling the next batch.
+        //
+        // schedule() is deliberately not awaited — batches must be able to
+        // process while the next one fills, which is the overlap this whole
+        // design exists for. But nothing bounded how far ahead the reader could
+        // get, so a source faster than the target (a DB cursor feeding a slower
+        // database, which is the normal case for a cross-engine move) piled
+        // batches up behind the semaphore until the memory monitor threw:
+        //
+        //   Streaming memory ceiling exceeded: 129MB > 128MB
+        //
+        // Read as a limit somebody should raise. It was not: it was a stream
+        // that had stopped being a stream. Rows were being read as fast as the
+        // source could produce them and held in `buffered` until their batch
+        // was written, so peak memory was governed by the SPEED DIFFERENCE
+        // between the two databases rather than by batchSize.
+        //
+        // `maxReadAhead` batches may be buffered beyond the ones in flight;
+        // past that the reader blocks here, and blocking here pauses the source
+        // itself when the source is a cursor. That is backpressure arriving
+        // where it was always supposed to.
+        await readAhead.wait();
       }
     }
 

package/lib/uploader/attach.js

@@ -8,26 +8,40 @@
 //   const { attachConfig } = require('@xeplr/actions');
 //   const xcfg = attachConfig({ service: 'xeplr-bi' });   // service = this app's name
 //   await xcfg.ready();
-//   xcfg.metaStore     // pass into uploads/loadMapping, uploader.rollback
-//   xcfg.ImportMeta    // BaseModel bound to xeplr_configs — reads/soft-deletes
-//   xcfg.SERVICE_NAME  // same string, for `.where('service', ...)` filters
+//   xcfg.metaStore       // pass into uploads/loadMapping, uploader.rollback
+//   xcfg.ImportMeta      // BaseModel bound to xeplr_configs — reads/soft-deletes
+//   xcfg.SERVICE_NAME    // same string, for `.where('service', ...)` filters
+//   xcfg.APPLICATION_ID  // the owning APP — see options.applicationId below
+//
+// An EMBEDDED package passes both, because they differ: the code is
+// @xeplr-workflow/api, the product is xeplr-bi.
+//
+//   attachConfig({ service: 'xeplr-workflow', applicationId: 'xeplr-bi' })
 //
 // import_meta's SCHEMA is identical for every app (see xeplr-db/config/
 // migrations/0001_import_meta.js + 0002_...) — there is nothing app-specific
 // about the ImportMeta model, so it lives here once instead of being
 // copy-pasted into every app's models/ directory.
-var { BaseModel, bootstrapConfigDb } = require('@xeplr/db');
+var { BaseModel, bootstrapConfigDb, resolveDbConnection, getApplicationId, assertApplicationIdColumns } = require('@xeplr/db');
 var { decrypt } = require('@xeplr/utils/isomorphic/crypto');
 var makeMetaStore = require('./meta-store-knex');
 
 var requiredEnv = [
   'XCFG_DB_NAME',                        // xeplr_configs db name
-  'XCFG_DB_CONNECTION_INFO_ENCRYPTED',   // xeplr_configs server login
+  // XCFG_DB_CONNECTION_INFO_ENCRYPTED is deliberately NOT here any more: the
+  // server login normally comes from the shared XEPLR_DB_CONNECTION, so
+  // demanding the xcfg-specific name would fail a correctly configured
+  // install. It still wins when set. Missing-ness is caught at the point of
+  // use by resolveDbConnection, which names both variables.
 ];
 
 class ImportMeta extends BaseModel {
   static get tableName() { return 'import_meta'; }
   static get idColumn() { return 'id'; }
+  // xeplr_configs is opened by EVERY app, so this is one of the few tables
+  // where the database is not the boundary and the column has to be — see
+  // BaseModel.applicationScoped.
+  static get applicationScoped() { return true; }
 }
 
 function attachConfig(options) {
@@ -35,6 +49,21 @@ function attachConfig(options) {
   var service = options.service;
   if (!service) throw new Error('attachConfig: options.service is required');
 
+  // The APPLICATION that owns rows written through this handle, as distinct
+  // from `service` (the package writing them). They differ precisely when a
+  // package is embedded: @xeplr-workflow/api running inside xeplr-bi is
+  // service='xeplr-workflow', applicationId='xeplr-bi'.
+  //
+  // Resolved LAZILY, at ready(), not here. Every app holds its instance in a
+  // module that is require()d at load time (db/xcfgSetup.js), which is before
+  // registerApplication() has run — reading it now would capture null in every
+  // app and quietly fall back to the package name, which is the precise bug
+  // this column exists to fix. An explicit options.applicationId still wins,
+  // for a caller that knows better than the process-global.
+  function resolveApplicationId() {
+    return options.applicationId || getApplicationId() || service;
+  }
+
   var _ready = null;
   var _xcfg = null;
   var _boundImportMeta = null;
@@ -43,10 +72,24 @@ function attachConfig(options) {
   function ready() {
     if (!_ready) {
       _ready = (async function() {
-        var login = JSON.parse(await decrypt(process.env.XCFG_DB_CONNECTION_INFO_ENCRYPTED, process.env.ENCRYPTION_KEY));
+        // XCFG_DB_CONNECTION_INFO_ENCRYPTED is an OVERRIDE — the connection
+        // normally comes from the shared XEPLR_DB_CONNECTION every xeplr
+        // service reads. XCFG_DB_NAME (which database) is untouched.
+        var login = JSON.parse(await decrypt(resolveDbConnection('XCFG_DB_CONNECTION_INFO_ENCRYPTED'), process.env.ENCRYPTION_KEY));
         _xcfg = await bootstrapConfigDb({ connection: login, database: process.env.XCFG_DB_NAME });
+
+        // xeplr_configs is THE shared store, so every table in it must be
+        // attributable to an application. Checked on each boot rather than
+        // trusted to the migration, because this database is written to by
+        // several codebases and a table added by any one of them without the
+        // column would go unnoticed until another app read it.
+        await assertApplicationIdColumns(_xcfg.db, { database: process.env.XCFG_DB_NAME });
         _boundImportMeta = ImportMeta.bindKnex(_xcfg.db);
-        _metaStore = makeMetaStore(_xcfg.db, { table: 'import_meta', service: service });
+        _metaStore = makeMetaStore(_xcfg.db, {
+          table: 'import_meta',
+          service: service,
+          applicationId: resolveApplicationId()
+        });
         return _xcfg;
       })();
     }
@@ -60,6 +103,9 @@ function attachConfig(options) {
   return {
     ready: ready,
     SERVICE_NAME: service,
+    // A getter, for the same reason resolveApplicationId is lazy: read at
+    // module load it would be the fallback, not the registered value.
+    get APPLICATION_ID() { return resolveApplicationId(); },
     get metaStore() { requireReady('metaStore'); return _metaStore; },
     get ImportMeta() { requireReady('ImportMeta'); return _boundImportMeta; },
     get db() { requireReady('db'); return _xcfg.db; }

package/lib/uploader/index.js

@@ -8,6 +8,13 @@
 //     connection,      // decrypted config object OR pool
 //     targetTable,
 //     primaryKeys,     // optional — enables UPSERT
+//     columns,         // optional — [{ name, type }] in the logical vocabulary
+//                       // (string|number|boolean|date|datetime|object|array).
+//                       // Supplied, types are DECLARED and the first batch is
+//                       // not sampled at all. Omitted, they are inferred from
+//                       // the first batch's values, which is right for a CSV
+//                       // and wrong wherever the source has a schema to ask.
+//                       // See beforeAll.
 //     movementId,      // required — the correlation key
 //     queue,           // required — a SqlQueue instance
 //     connectionName,  // key used in queue.connections; defaults to targetTable
@@ -21,6 +28,9 @@
 //     errorTable,      // defaults to `<targetTable>_import_errors`
 //     batchDir,        // where NDJSON files spool to. Defaults to os.tmpdir()/xeplr-actions
 //     keepBatchFiles,  // default false — clean up NDJSON files after success
+//     onProgress,      // optional ({rowsRead, batches}) => void, called per batch.
+//                       // Rows READ, not rows confirmed written — see onBatch.
+//                       // Throwing in it cannot fail the movement.
 //   })
 //
 // EXECUTION MODEL (production-critical — read this):
@@ -55,6 +65,7 @@ async function upload(opts) {
   var driver = opts.driver;
   var startedAt = Date.now();
 
+  var rowsSeen = 0;
   var poolIsExternal = looksLikePool(opts.connection);
   var pool = poolIsExternal ? opts.connection : await driver.connect(opts.connection);
 
@@ -75,15 +86,33 @@ async function upload(opts) {
       maxMemoryMB:  opts.spoolMaxMemoryMB || 128,
 
       target: {
-        // Bootstrap: first batch is on disk. Infer types from it, run DDL,
-        // reconcile against the target's existing schema, then compute the
-        // safe batch size for INSERT.
+        // Bootstrap: first batch is on disk. Work out the column types, run
+        // DDL, reconcile against the target's existing schema, then compute
+        // the safe batch size for INSERT.
         beforeAll: async function(ctx) {
           runDir = ctx.batchDir;
-          var sample = await readBatchFile(ctx.firstBatchPath);
-          var scanLimit = Math.min(sample.length, opts.firstBatchScanRows);
-          var inferred = inferColumns(sample.slice(0, scanLimit));
-          columns = await runBootstrap(driver, pool, opts.targetTable, inferred, opts.primaryKeys);
+          var declared;
+          if (opts.columns && opts.columns.length) {
+            // DECLARED, so nothing is guessed.
+            //
+            // Inference reads VALUES, which is the only option for a CSV and
+            // the wrong one when the source has a schema you can just ask for.
+            // The failures are quiet and permanent: Postgres hands `numeric`
+            // to node as a STRING, so a money column arrives as strings, is
+            // inferred as text, and the target column is created TEXT — you
+            // find out when somebody tries to SUM it. An all-null sample types
+            // as string for the same reason. A zero-padded code becomes a
+            // number and loses its leading zeros.
+            //
+            // The guess is fine when it is all you have. It should not be what
+            // happens when it is not.
+            declared = opts.columns;
+          } else {
+            var sample = await readBatchFile(ctx.firstBatchPath);
+            var scanLimit = Math.min(sample.length, opts.firstBatchScanRows);
+            declared = inferColumns(sample.slice(0, scanLimit));
+          }
+          columns = await runBootstrap(driver, pool, opts.targetTable, declared, opts.primaryKeys);
           effectiveBatchSize = Math.min(opts.batchSize, safeBatchSize(columns.length, opts.maxParamsPerBatch));
           await opts.metaStore.recordStart(opts.movementId, {
             targetTable:   opts.targetTable,
@@ -101,13 +130,35 @@ async function upload(opts) {
         // addToQueue awaits when queue is over its 200MB backpressure, which
         // in turn awaits spool's next batch — the whole pipeline throttles
         // correctly, but disk (not memory) is the buffer.
-        onBatch: async function(rows, _meta) {
+        onBatch: async function(rows, meta) {
           if (!columns) throw new Error('uploader: onBatch before beforeAll — spool contract violated');
           for (var off = 0; off < rows.length; off += effectiveBatchSize) {
             var chunk = rows.slice(off, off + effectiveBatchSize);
-            await enqueueBatch(opts, chunk, columns);
+            // A NAME FOR THIS BATCH, carried all the way to the failure.
+            //
+            // Without one, "20 rows dropped" is unactionable: there is nothing
+            // to look up, nothing to re-run, nothing to tell somebody over the
+            // phone. b<spool batch>.<chunk> is stable for a given run and
+            // enough for a human to find the rows again in the source.
+            var batchRef = 'b' + (meta ? meta.batchIndex : totalBatches) +
+              '.' + Math.floor(off / effectiveBatchSize);
+            await enqueueBatch(opts, chunk, columns, batchRef);
             totalBatches++;
           }
+          rowsSeen += rows.length;
+          // WHAT HAS BEEN READ, not what has landed — the queue is still
+          // draining behind this. Deliberately so: for a long movement the only
+          // honest live number is "rows so far", and waiting for the write to
+          // confirm would report nothing for minutes.
+          //
+          // No total, because for a query source there isn't one to know
+          // without running it twice. A caller that wants a percentage has to
+          // supply its own denominator.
+          if (opts.onProgress) {
+            try {
+              opts.onProgress({ rowsRead: rowsSeen, batches: meta ? meta.batchIndex + 1 : totalBatches });
+            } catch (e) { /* a reporting failure must not fail the movement */ }
+          }
         }
       }
     });
@@ -175,17 +226,38 @@ async function upload(opts) {
 
 // Rollback a movement — drop from main + error tables, clear meta row.
 // Graceful queue abort first (in-flight SQLs allowed to settle).
+//
+// Refuses outright for any movement recorded with primaryKeys (i.e. it ran
+// as an upsert). UPSERT's ON CONFLICT/ON DUPLICATE KEY/MERGE stamps
+// __xeplr_movement_id__ onto every row it MATCHES, not just the ones it
+// creates (see buildInsertSql in each driver) — so a pre-existing row that
+// was merely updated ends up tagged with this run's movementId too. Deleting
+// by that column can't tell "created by this run" from "already existed and
+// got touched by it" apart after the fact, so there is no safe partial
+// rollback to fall back to — the whole operation is refused.
 async function rollback(opts) {
   if (!opts || !opts.movementId)  throw new Error('rollback: movementId is required');
   if (!opts.driver)               throw new Error('rollback: driver is required');
   if (!opts.connection)           throw new Error('rollback: connection is required');
   if (!opts.targetTable)          throw new Error('rollback: targetTable is required');
 
+  if (opts.metaStore && opts.metaStore.getById) {
+    var meta = await opts.metaStore.getById(opts.movementId);
+    if (meta && meta.primary_keys && meta.primary_keys.length) {
+      throw new Error(
+        'rollback: this movement used upsert (primary keys: ' + meta.primary_keys.join(', ') + ') — ' +
+        'rolling back could delete rows that already existed and were only updated, not created. ' +
+        'Rollback is not supported for upsert movements.'
+      );
+    }
+  }
+
   if (opts.queue) {
     opts.queue.abort(opts.movementId, 'rollback');
     await opts.queue.drain();
   }
 
+  var rowsSeen = 0;
   var poolIsExternal = looksLikePool(opts.connection);
   var pool = poolIsExternal ? opts.connection : await opts.driver.connect(opts.connection);
 
@@ -260,7 +332,7 @@ async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
   return reconciled;
 }
 
-async function enqueueBatch(opts, rows, columns) {
+async function enqueueBatch(opts, rows, columns, batchRef) {
   var driver     = opts.driver;
   var target     = opts.targetTable;
   var movementId = opts.movementId;
@@ -277,6 +349,8 @@ async function enqueueBatch(opts, rows, columns) {
     meta: {
       rows:       rows,
       rowCount:   rows.length,
+      // Travels with the statement so a drop can name which batch it was.
+      batchRef:   batchRef || null,
       rowsToSql:  rowsToSql,
       errorTable: opts.errorTable
     }

package/lib/uploader/meta-store-knex.js

@@ -11,7 +11,11 @@
 // app that's calling knows its own name, no reason to make callers thread it
 // through every recordStart/recordEnd call):
 //   const makeStore = require('@xeplr/actions/lib/uploader/meta-store-knex');
-//   const store = makeStore(knex, { table: 'import_meta', service: 'xeplr-bi' });
+//   const store = makeStore(knex, {
+//     table: 'import_meta',
+//     service: 'xeplr-workflow',     // the PACKAGE writing the row
+//     applicationId: 'xeplr-bi'      // the PRODUCT that owns it
+//   });
 //   await upload({ ..., metaStore: store });
 //
 // Records NO secrets: `connection_key` is an opaque label the consumer chose,
@@ -19,13 +23,24 @@
 // plan. `details` (app-specific context — filename, connectionId, who ran
 // it) and `mtId1`-`mtId4` (tenant scoping, same column names as BaseModel)
 // are set once at recordStart and never touched by recordEnd — this table is
-// shared across apps, so every read/write is scoped by `service` too.
+// shared across apps, so every read/write is scoped by `applicationId` (which
+// PRODUCT owns the row) and `service` (which PACKAGE wrote it). The two are
+// the same string for a standalone deployment and differ for an embedded one,
+// which is the whole reason both exist.
 
 module.exports = function makeKnexMetaStore(knex, options) {
   if (!knex) throw new Error('meta-store-knex: a knex instance is required');
   if (!options || !options.service) throw new Error('meta-store-knex: options.service is required');
   var TABLE = options.table || 'import_meta';
   var SERVICE = options.service;
+  // This store talks RAW KNEX, so it gets none of BaseModel's automatic
+  // applicationId stamping or filtering — every statement below has to carry
+  // it by hand. That is the price of not going through the model, and the
+  // reason it is required here rather than defaulted: a store that silently
+  // wrote unattributed rows into the shared control plane would be worse than
+  // one that refuses to be constructed.
+  if (!options.applicationId) throw new Error('meta-store-knex: options.applicationId is required — import_meta is shared across apps');
+  var APPLICATION_ID = options.applicationId;
 
   return {
     // Most-recent movement for a target table (resume / dedupe / audit).
@@ -35,12 +50,24 @@ module.exports = function makeKnexMetaStore(knex, options) {
       var targetTable = opts && opts.targetTable;
       if (!targetTable) return null;
       var row = await knex(TABLE)
-        .where({ service: SERVICE, target_table: targetTable })
+        .where({ applicationId: APPLICATION_ID, service: SERVICE, target_table: targetTable })
         .orderBy('started_at', 'desc')
         .first();
       return row || null;   // pg returns jsonb columns already parsed
     },
 
+    // Fetch one movement's own row by id — used by rollback() to check what
+    // the movement actually was (e.g. did it use primaryKeys/upsert) before
+    // deciding whether deleting by movementId is safe.
+    async getById(movementId) {
+      // Scoped even though `id` is the primary key: rollback() decides from
+      // this row whether deleting by movementId is safe, and answering that
+      // question from ANOTHER application's row is the worst possible outcome
+      // here. A miss returns null and rollback declines, which is correct.
+      var row = await knex(TABLE).where({ id: movementId, applicationId: APPLICATION_ID }).first();
+      return row || null;   // pg returns jsonb columns already parsed
+    },
+
     // Open the movement row in 'running' state. Upsert on id so a re-run /
     // resume of the same movementId overwrites its prior 'running' row.
     async recordStart(movementId, plan) {
@@ -48,6 +75,7 @@ module.exports = function makeKnexMetaStore(knex, options) {
       await knex(TABLE)
         .insert({
           id:             movementId,
+          applicationId:  APPLICATION_ID,
           service:        SERVICE,
           mtId1:          plan.mtId1 || null,
           mtId2:          plan.mtId2 || null,
@@ -83,7 +111,7 @@ module.exports = function makeKnexMetaStore(knex, options) {
         }
       }
       if (!touched) return;
-      await knex(TABLE).where({ id: movementId }).update(patch);
+      await knex(TABLE).where({ id: movementId, applicationId: APPLICATION_ID }).update(patch);
     },
 
     // Close the movement. Upserts: the empty-source path calls recordEnd
@@ -106,7 +134,7 @@ module.exports = function makeKnexMetaStore(knex, options) {
         patch.meta = JSON.stringify({ mainDeleted: result.mainDeleted, errorDeleted: result.errorDeleted });
       }
 
-      var updated = await knex(TABLE).where({ id: movementId }).update(patch);
+      var updated = await knex(TABLE).where({ id: movementId, applicationId: APPLICATION_ID }).update(patch);
       if (!updated) {
         // No prior recordStart (empty-source path — beforeAll never fires) —
         // this is the only chance to stamp service/tenant/plan context, or
@@ -115,6 +143,7 @@ module.exports = function makeKnexMetaStore(knex, options) {
         // recordEnd call).
         var fallback = {
           id: movementId,
+          applicationId: APPLICATION_ID,
           service: SERVICE,
           mtId1: result.mtId1 || null,
           mtId2: result.mtId2 || null,

package/lib/uploader/meta-store-noop.js

@@ -5,6 +5,7 @@
 
 module.exports = {
   async getLast(_opts) { return null; },
+  async getById(_movementId) { return null; },
   async recordStart(_uploadId, _plan) { /* no-op */ },
   async recordProgress(_uploadId, _delta) { /* no-op */ },
   async recordEnd(_uploadId, _result) { /* no-op */ }

package/lib/uploader/reconcile.js

@@ -30,7 +30,18 @@ var PG_TO_LOGICAL = {
   'boolean':                     'boolean',
   'timestamp with time zone':    'datetime',
   'timestamp without time zone': 'datetime',
-  'date':                        'datetime',
+  // A DATE IS NOT A TIMESTAMP, and mapping it to one moves it.
+  //
+  // This said 'datetime', so a source `date` was created as TIMESTAMPTZ in the
+  // target and the value acquired midnight in the server's zone. From +05:30,
+  // 2026-01-05 stored as 2026-01-04T18:30:00Z — a business date a day earlier
+  // than the one that was moved, and every report grouped by it off by one for
+  // half the day.
+  //
+  // Reachable only via a declared schema (inference sees an ISO string and
+  // still says 'datetime', correctly — it cannot know), so nothing that infers
+  // changes behaviour. See upload()'s `columns` option.
+  'date':                        'date',
   'time with time zone':         'datetime',
   'time without time zone':      'datetime',
   'json':                        'object',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.1.1",
+  "version": "1.1.6",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "files": [
@@ -31,13 +31,18 @@
     "@xeplr/schema-handler": "^1.0.0"
   },
   "peerDependencies": {
+    "@duckdb/node-api": "1.5.5-r.3",
+    "@xeplr/email": "^1.0.0",
     "@xeplr/utils": "^1.0.0",
     "csv-parse": "^5.0.0",
     "exceljs": "^4.0.0",
+    "imapflow": "^1.0.0",
     "knex": "^3.0.0",
+    "mailparser": "^3.0.0",
     "mongodb": "^6.0.0",
     "mssql": "^11.0.0",
     "mysql2": "^3.0.0",
+    "nodemailer": "^8.0.2",
     "pg": "^8.0.0",
     "saxes": "^6.0.0",
     "ssh2-sftp-client": "^11.0.0",
@@ -47,6 +52,9 @@
     "@xeplr/utils": {
       "optional": true
     },
+    "@duckdb/node-api": {
+      "optional": true
+    },
     "knex": {
       "optional": true
     },
@@ -74,11 +82,24 @@
     "csv-parse": {
       "optional": true
     },
+    "imapflow": {
+      "optional": true
+    },
+    "mailparser": {
+      "optional": true
+    },
+    "nodemailer": {
+      "optional": true
+    },
     "ssh2-sftp-client": {
       "optional": true
+    },
+    "@xeplr/email": {
+      "optional": true
     }
   },
   "devDependencies": {
+    "@duckdb/node-api": "^1.5.5-r.3",
     "csv-parse": "^5.0.0",
     "express": "^4.0.0",
     "mssql": "^12.7.0",