@xeplr/actions 1.1.14 → 1.1.16

package/README.md

@@ -72,7 +72,7 @@ var moved = await actions.runAction(actions.builtins.dbMove, {
 | `ActionMissingDependencyError` | Thrown by `register()` for a missing `requires` module. |
 | `TransientError` | For an action to signal "retry me". Nothing in this package (or `@xeplr/jobs`) acts on it yet. |
 | `builtins` | The built-in action modules — see [Built-in actions](#built-in-actions). |
-| `uploader` | `{ upload, rollback, inferColumns, inferColumnType, makeMetaStore }` — see [Uploader](#uploader). |
+| `uploader` | `{ upload, rollback, wasCommitted, inferColumns, inferColumnType, makeMetaStore }` — see [Uploader](#uploader). |
 | `inferColumns(rows)` | `[{ name, type }]` from sample rows. |
 | `streaming` | `{ spool, resume, readBatchFile, checkpoint }` — rows to rotating NDJSON batch files with ordered hooks and resumable checkpoints. |
 | `drivers.db` | `{ getDriver(type), checkDriverRequires, SUPPORTED }` — `postgres`, `mysql`, `mssql`, `mongo`, `duckdb`. |
@@ -141,6 +141,7 @@ The db actions `db-fetch`, `db-push` and `db-procedure` accept a connection two
 | `writeMode` | `append` (default), `replace` (TRUNCATE the target first if it exists, DELETE if TRUNCATE fails), `upsert` (**requires** `primaryKeys`). |
 | `window` | `{ column, from, to }` → `column >= from AND column < to` (a null edge is left out). `{ columns: [{ column, from }] }` → OR of `column >= from` (null `from`s left out). For a procedure: `{ from, to, fromParam, toParam }`, passed as parameters. A window with none of `column`, `columns`, `fromParam`, `toParam` is treated as no window. |
 | `where` | Raw SQL condition, ANDed with the window in parentheses, run on the source. Not parameterised. Ignored for `mode: 'procedure'`. |
+| `staged` | `true`: write into `<targetTable>__xeplr_staging` and move it into the target in one transaction at the end (see [Staged loads](#staged-loads)). DuckDB targets only. Default `false`. |
 | `movementId` | Correlation key. Default `system.occurrenceId`, else `mv_<id>`. |
 | `batchSize`, `concurrency` | Default 5000 and 4. |
 
@@ -184,13 +185,17 @@ await actions.uploader.upload({
   columns,                // optional [{ name, type }] — declared; skips inference
   batchSize: 5000, firstBatchScanRows: 1000,
   metaStore, mtId1, mtId2, mtId3, mtId4, details,
-  errorTable, batchDir, keepBatchFiles, onProgress
+  errorTable, batchDir, keepBatchFiles, onProgress,
+  signal,                 // optional AbortSignal: stop on request
+  staged                  // optional: see Staged loads
 })
-// → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, durationMs }
+// → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, merged, durationMs }
 ```
 
 `onProgress({ rowsRead, batches, rowsWritten, phase? })` fires after each batch is read, with `rowsWritten` from the write queue's own count, so a caller can show both and see whether writing keeps up. After the read, while the queue drains, it fires again each time the written count moves (`phase: 'writing'`), never on a bare timer, so a stuck write goes quiet.
 
+`signal` (an `AbortSignal`, passed to db-move as `system.signal`) stops a running movement: the writes still queued are dropped at once (in-flight ones settle), the read stops at the next batch, and it ends with an error whose `code` is `'STOPPED'` (the runner keeps `error.code`), neither a failure nor a success. What it wrote stays, under its `movementId`, for the caller to roll back or keep.
+
 Rows are spooled to NDJSON files on disk between source and database. From the first batch it creates, if missing: the target table, `<targetTable>_import_errors` (`movement_id`, `row_num`, `error_description`, `underlying_sql`, `raw_row`, `recorded_at`), a `__xeplr_movement_id__` column, and a unique index on `primaryKeys`. Server dialects also add `__xeplr_id__`; DuckDB does not.
 
 Logical types and what each driver creates:
@@ -206,6 +211,18 @@ Logical types and what each driver creates:
 
 Inference (`inferColumns`): any array → `array`; any object → `object`; all `Date` or ISO-date strings → `datetime`; all booleans → `boolean`; all numbers → `number`; anything else, or all null → `string`.
 
+### Staged loads
+
+With `staged: true` a load **never writes into its target while it runs**. Batches land in `<targetTable>__xeplr_staging` (made empty, shaped like the target, at the start). When every batch is in, one transaction moves them into the target and writes a row to `__xeplr_loads` (`movement_id`, `target_table`, `inserted`, `replaced`, `committed_at`); then the staging table is dropped. `merged` in the result is `{ inserted, replaced }`; `onProgress` reports `phase: 'merging'` just before the move.
+
+- **Stopped, failed, aborted or killed before the move**: the target is untouched. The staging table is dropped (or, after a crash, by the next load's start). Undoing it costs what the load wrote, never what the target holds; there is nothing to roll back.
+- **Crash during the move**: the transaction is discarded on reopen and the target is as it was.
+- **Did it land?** `uploader.wasCommitted({ driver, connection, movementId })` reads the ledger row, written in the same transaction as the move: a row means it landed, `null` means nothing did. No scan of the target.
+- **Upsert** replaces target rows whose key is staged (`DELETE … USING` staging, then `INSERT`). Within one load the last row staged for a key wins, as under `ON CONFLICT DO UPDATE`; rows with a NULL key match nothing and are kept. **No unique index**: a staged load drops `<targetTable>_upsert_uniq` and never creates it. DuckDB's DELETE through an index that a killed load left out of step with its rows fails with "Failed to delete all rows from index", and that failure invalidates the whole database.
+- **Cost** is one statement at the end. Measured with 50 lakh staged rows into 50 crore: append 0.3 s (independent of the target's size), upsert 0.4–7 s depending on how widely the replaced keys are spread.
+
+A driver opts in by implementing `prepareStaging`, `mergeStaged`, `dropStaging` and `committedMovement`; `upload` refuses `staged` for one that does not.
+
 `uploader.rollback({ movementId, driver, connection, targetTable, metaStore?, queue? })` aborts the movement's queued work, deletes its rows from the target and error tables by `__xeplr_movement_id__` / `movement_id`, and records `rolled-back`.
 
 ### Formats and file sources
@@ -287,6 +304,7 @@ await driver.query(pool, userSql, params)          // reads only
 | `db-fetch` streams to disk by default; `streaming_mode: false` only for small reads. | Bounded memory at both ends for cross-engine replication. |
 | **Half-open windows** (`from <= x < to`), no +1 second. | Consecutive windows meet exactly; closed windows move the boundary row twice. |
 | `db-move` `upsert` without `primaryKeys` is refused, not downgraded to append. | A silent append is a duplicate every run, found a week later. |
+| A **staged** load touches its target only in one transaction at the end, and keeps no unique index. | Undoing a stopped load must cost what it wrote, not a rebuild of the table it was adding to; and a DELETE through a damaged DuckDB index invalidates the database. |
 | `replace` truncates before reading; on failure the target stays empty. | Accepted trade against keeping a second copy of the data. |
 | A procedure call with a window but no `fromParam`/`toParam` is refused (`db-procedure`, and the shared builder `db-move` uses). | A procedure cannot be filtered from outside; dropping the window would reprocess its whole history every run while reporting success. |
 | `db-move` uses the source table's declared types for `mode: 'table'`. | Inference is lossy: Postgres `numeric` arrives as a string and would create a TEXT column; an all-null sample becomes text; `'007'` loses its zeros. |
@@ -312,8 +330,8 @@ node --test test/uploader/reconcile.test.js
 
 | needs nothing | needs a database |
 |---|---|
-| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
-| `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
+| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
+| `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js`, `test/staged-load.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
 | | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
 | | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |
 

package/lib/builtins/db/move.js

@@ -94,6 +94,11 @@ module.exports = {
     { name: 'primaryKeys',     type: 'array',  order: 12, group: 'Write options',
       description: 'Target column names for UPSERT. Required when writeMode="upsert".' },
 
+    { name: 'staged',          type: 'boolean', default: false, order: 12.5, group: 'Write options',
+      description: 'Write into a staging table and move it into the target in one transaction at ' +
+                   'the end, so a movement that stops or fails midway never touches the target. ' +
+                   'DuckDB targets only.' },
+
     { name: 'window',          type: 'object', order: 13, group: 'Advanced',
       description: 'One window: { column, from, to }. Half-open — from <= x < to. ' +
                    'Omitted = no filter. For mode="procedure", also needs { fromParam, toParam }.' },
@@ -164,6 +169,9 @@ module.exports = {
         { phase: 'start', event: 'columns', count: (input.columns || []).length });
       system.log('Write mode: ' + describeWriteMode(writeMode, primaryKeys),
         { phase: 'start', event: 'write_mode', writeMode: writeMode, primaryKeys: primaryKeys || [] });
+      if (input.staged) system.log('Staged: rows collect in "' + input.targetTable + '__xeplr_staging" and move into "' +
+        input.targetTable + '" in one step at the end — a stop or failure before then leaves "' + input.targetTable + '" untouched',
+        { phase: 'start', event: 'staged', stagingTable: input.targetTable + '__xeplr_staging' });
       system.log('Window: ' + describeWindow(input.window),
         { phase: 'start', event: 'window',
           // Bounds are CONFIGURATION — a date range somebody chose — not rows
@@ -304,7 +312,10 @@ module.exports = {
         // Passed through from the caller's `system`, not from `input` — a
         // progress reporter is a function, and an action's input is data that
         // has to survive being stored in a job row.
-        onProgress:     system.onProgress || null
+        onProgress:     system.onProgress || null,
+        // An AbortSignal from the caller: stop reading, drop the queued writes.
+        signal:         system.signal || null,
+        staged:         Boolean(input.staged)
       });
 
       // Every count upload() has, not just the headline one — "read" and
@@ -318,6 +329,10 @@ module.exports = {
           bits.push(result.dropped + ' dropped' +
             (droppedGroups ? ' in ' + droppedGroups + ' failed batch(es) — see the BATCH … FAILED lines for which' : ''));
         }
+        if (result.merged) {
+          bits.push('moved into the table in one step (' + result.merged.inserted + ' in' +
+            (result.merged.replaced ? ', ' + result.merged.replaced + ' replaced' : '') + ')');
+        }
         bits.push((result.totalBatches || 0) + ' batch' + (result.totalBatches === 1 ? '' : 'es'));
         bits.push(((result.durationMs || 0) / 1000).toFixed(1) + 's');
         // THE LINE THE WHOLE LOG EXISTS FOR. Every number a discrepancy

package/lib/drivers/db/duckdb.js

@@ -738,6 +738,131 @@ async function rollbackMovement(pool, tableName, movementId) {
   return result;
 }
 
+// ── staged loads ─────────────────────────────────────────────────────────
+//
+// A STAGED load never writes into its target while it runs. Its batches land
+// in `<target>__xeplr_staging`; when every batch is in, ONE transaction moves
+// them into the target and records the load in `__xeplr_loads`. So a load
+// that stops, fails or is killed midway has touched only the staging table:
+// undoing it costs what IT wrote, never what the target already held, and no
+// rollback of the target is ever needed.
+//
+// DuckDB commits a transaction whole or not at all, so a crash during the
+// move leaves the target as it was. The ledger row is written in that same
+// transaction: if it is there the load landed, if it is not nothing did —
+// answered without scanning the target (see committedMovement).
+//
+// NO UNIQUE INDEX. A staged upsert matches keys in the move itself, so the
+// upsert index is dropped rather than created. It was the index a copy killed
+// mid-write left out of step with its rows, and a DELETE through such an index
+// fails with "Failed to delete all rows from index" — fatally, invalidating
+// the whole database.
+
+var STAGING_SUFFIX = '__xeplr_staging';
+var LOADS_LEDGER   = '__xeplr_loads';
+
+function stagingTableFor(tableName) { return tableName + STAGING_SUFFIX; }
+
+/** Run fn on ONE connection — a transaction's statements must not be spread across the pool. */
+async function withConnection(pool, fn) {
+  var entry = entryOf(pool);
+  var conn = await acquire(entry);
+  try {
+    return await fn(conn);
+  } finally {
+    release(entry, conn);
+  }
+}
+
+/**
+ * A fresh, empty staging table shaped exactly like the target (which must
+ * exist). A staging table already there is a load that died before its move,
+ * and is dropped: nothing in it ever reached the target.
+ */
+async function prepareStaging(pool, tableName) {
+  var staging = stagingTableFor(tableName);
+  await query(pool, 'DROP INDEX IF EXISTS ' + quoteIdent(tableName + '_upsert_uniq'));
+  await query(pool, 'DROP TABLE IF EXISTS ' + quoteIdent(staging));
+  await query(pool, 'CREATE TABLE ' + quoteIdent(staging) + ' AS SELECT * FROM ' + quoteIdent(tableName) + ' LIMIT 0');
+  return staging;
+}
+
+async function dropStaging(pool, tableName) {
+  await query(pool, 'DROP TABLE IF EXISTS ' + quoteIdent(stagingTableFor(tableName)));
+}
+
+/**
+ * Move a load's staged rows into its target, in one transaction with its
+ * ledger row, then drop the staging table.
+ *
+ * With primaryKeys it is an upsert: target rows whose key is staged are
+ * replaced. Within one load the last row staged for a key wins, as it did
+ * under ON CONFLICT DO UPDATE; a row with a NULL key matches nothing, and is
+ * kept, as ON CONFLICT kept it.
+ *
+ * Returns { inserted, replaced }.
+ */
+async function mergeStaged(pool, opts) {
+  var target = quoteIdent(opts.targetTable);
+  var staging = quoteIdent(stagingTableFor(opts.targetTable));
+  var ledger = quoteIdent(LOADS_LEDGER);
+  var pks = (opts.primaryKeys && opts.primaryKeys.length) ? opts.primaryKeys : null;
+  var cols = [INTERNAL_MOVEMENT_COL].concat(opts.columns.map(function(c) { return c.name; }))
+    .map(quoteIdent).join(', ');
+
+  var source = staging;
+  if (pks) {
+    var keys = pks.map(quoteIdent);
+    source = '(SELECT ' + cols + ' FROM ' + staging +
+      ' QUALIFY row_number() OVER (PARTITION BY ' + keys.join(', ') + ' ORDER BY rowid DESC) = 1' +
+      ' OR ' + keys.map(function(k) { return k + ' IS NULL'; }).join(' OR ') + ')';
+  }
+
+  var counts = await withConnection(pool, async function(conn) {
+    await conn.run('CREATE TABLE IF NOT EXISTS ' + ledger +
+      ' (movement_id VARCHAR, target_table VARCHAR, inserted BIGINT, replaced BIGINT, committed_at TIMESTAMPTZ)');
+    await conn.run('BEGIN TRANSACTION');
+    try {
+      var replaced = 0;
+      if (pks) {
+        var del = await conn.run('DELETE FROM ' + target + ' AS t USING ' + staging + ' AS s WHERE ' +
+          pks.map(function(k) { return 't.' + quoteIdent(k) + ' = s.' + quoteIdent(k); }).join(' AND '));
+        replaced = Number(del.rowsChanged) || 0;
+      }
+      var ins = await conn.run('INSERT INTO ' + target + ' (' + cols + ') SELECT ' + cols + ' FROM ' + source);
+      var inserted = Number(ins.rowsChanged) || 0;
+      await conn.run('INSERT INTO ' + ledger + ' VALUES ($1, $2, $3, $4, now())',
+        [opts.movementId, opts.targetTable, inserted, replaced]);
+      await conn.run('COMMIT');
+      return { inserted: inserted, replaced: replaced };
+    } catch (err) {
+      await conn.run('ROLLBACK').catch(function() {});
+      throw err;
+    }
+  });
+
+  // After the commit, and allowed to fail: the load has landed, and the next
+  // load drops a leftover staging table anyway.
+  await dropStaging(pool, opts.targetTable).catch(function() {});
+  return counts;
+}
+
+/**
+ * Did this movement's staged load reach its target? The ledger row, or null.
+ * The answer a run that ended without saying (a killed process) needs.
+ */
+async function committedMovement(pool, movementId) {
+  var r;
+  try {
+    r = await query(pool, 'SELECT movement_id, target_table, inserted, replaced, committed_at FROM ' +
+      quoteIdent(LOADS_LEDGER) + ' WHERE movement_id = $1 LIMIT 1', [movementId]);
+  } catch (err) {
+    if (/does not exist/i.test(String(err.message))) return null;
+    throw err;
+  }
+  return r.rows[0] || null;
+}
+
 module.exports = {
   requires: ['@duckdb/node-api'],
 
@@ -766,6 +891,13 @@ module.exports = {
 
   rollbackMovement: rollbackMovement,
 
+  stagingTableFor:   stagingTableFor,
+  prepareStaging:    prepareStaging,
+  dropStaging:       dropStaging,
+  mergeStaged:       mergeStaged,
+  committedMovement: committedMovement,
+  LOADS_LEDGER:      LOADS_LEDGER,
+
   INTERNAL_ID_COL:       INTERNAL_ID_COL,
   INTERNAL_MOVEMENT_COL: INTERNAL_MOVEMENT_COL,
 

package/lib/runner.js

@@ -109,6 +109,8 @@ function finish(start, status, output, err) {
     if (err.stack) e.stack = err.stack;
     if (err.details) e.details = err.details;
     if (err.actionName) e.actionName = err.actionName;
+    // Kept, so a caller can tell a movement STOPPED on request from one that failed.
+    if (err.code) e.code = err.code;
   }
   return { status: status, output: output, error: e, durationMs: Date.now() - start };
 }

package/lib/uploader/index.js

@@ -28,6 +28,11 @@
 //     errorTable,      // defaults to `<targetTable>_import_errors`
 //     batchDir,        // where NDJSON files spool to. Defaults to os.tmpdir()/xeplr-actions
 //     keepBatchFiles,  // default false — clean up NDJSON files after success
+//     staged,          // default false. true: batches land in <targetTable>__xeplr_staging
+//                       // and move into the target in ONE transaction at the end, so a
+//                       // load that stops or dies midway never touched the target.
+//                       // Needs a driver with mergeStaged (DuckDB). See the driver's
+//                       // "staged loads" section.
 //     onProgress,      // optional ({rowsRead, batches}) => void, called per batch.
 //                       // Rows READ, not rows confirmed written — see onBatch.
 //                       // Throwing in it cannot fail the movement.
@@ -76,6 +81,18 @@ async function upload(opts) {
 
   var spoolResult = null;
   var runDir = null;
+  // A staged load's staging table, once made — and whether its rows have moved
+  // into the target. Anything else at the end means they never will: dropped.
+  var staged = false;
+  var merged = null;
+
+  // STOPPING ON REQUEST (opts.signal, an AbortSignal). The writes still queued
+  // are dropped the moment it fires — in-flight ones settle — and the read
+  // stops at the next batch. What was written stays, identified by
+  // movementId, for the caller to roll back or keep.
+  if (opts.signal && opts.signal.aborted) throw stoppedError();
+  var onStop = opts.signal ? function() { opts.queue.abort(opts.movementId, 'stopped'); } : null;
+  if (onStop) opts.signal.addEventListener('abort', onStop, { once: true });
 
   try {
     var summary = await spool(opts.source, {
@@ -112,7 +129,11 @@ async function upload(opts) {
             var scanLimit = Math.min(sample.length, opts.firstBatchScanRows);
             declared = inferColumns(sample.slice(0, scanLimit));
           }
-          columns = await runBootstrap(driver, pool, opts.targetTable, declared, opts.primaryKeys);
+          columns = await runBootstrap(driver, pool, opts.targetTable, declared, opts.primaryKeys, opts.staged);
+          if (opts.staged) {
+            opts.writeTable = await driver.prepareStaging(pool, opts.targetTable);
+            staged = true;
+          }
           effectiveBatchSize = Math.min(opts.batchSize, safeBatchSize(columns.length, opts.maxParamsPerBatch));
           await opts.metaStore.recordStart(opts.movementId, {
             targetTable:   opts.targetTable,
@@ -131,6 +152,7 @@ async function upload(opts) {
         // in turn awaits spool's next batch — the whole pipeline throttles
         // correctly, but disk (not memory) is the buffer.
         onBatch: async function(rows, meta) {
+          if (opts.signal && opts.signal.aborted) throw stoppedError();
           if (!columns) throw new Error('uploader: onBatch before beforeAll — spool contract violated');
           for (var off = 0; off < rows.length; off += effectiveBatchSize) {
             var chunk = rows.slice(off, off + effectiveBatchSize);
@@ -207,9 +229,27 @@ async function upload(opts) {
     } finally {
       if (drainWatch) drainWatch.stop();
     }
+    // Stopped while the queue drained: not a finished movement.
+    if (opts.signal && opts.signal.aborted) throw stoppedError();
 
     var stats = opts.queue.stats(opts.movementId) || {};
     var status = stats.aborted ? 'aborted' : 'completed';
+
+    // THE ONE STEP THAT TOUCHES THE TARGET, for a staged load. Not for an
+    // aborted one: its rows are dropped with the staging table, in finally.
+    if (staged && !stats.aborted) {
+      if (opts.onProgress) {
+        try {
+          opts.onProgress({ rowsRead: rowsSeen, batches: totalBatches, rowsWritten: writtenSoFar(opts), phase: 'merging' });
+        } catch (e) { /* reporting must not fail the movement */ }
+      }
+      merged = await driver.mergeStaged(pool, {
+        targetTable: opts.targetTable,
+        columns:     columns,
+        primaryKeys: opts.primaryKeys,
+        movementId:  opts.movementId
+      });
+    }
     await opts.metaStore.recordEnd(opts.movementId, {
       status:       status,
       totalRows:    summary.totalRows,
@@ -227,18 +267,33 @@ async function upload(opts) {
       completed:    stats.completed || 0,
       dropped:      stats.dropped || 0,
       aborted:      !!stats.aborted,
+      // Staged only: what the move put in the target — { inserted, replaced }.
+      merged:       merged,
       durationMs:   Date.now() - startedAt
     };
   } finally {
+    // A staged load that did not move its rows: stopped, failed, aborted.
+    // Its staging table is all it wrote, and dropping it is the whole undo.
+    if (staged && !merged && pool) {
+      try { await driver.dropStaging(pool, opts.targetTable); } catch (_) {}
+    }
     // Best-effort cleanup: delete the NDJSON spool directory unless caller
     // asked to keep it (e.g. for later resume / audit).
     if (runDir && !opts.keepBatchFiles) {
       try { await fsp.rm(runDir, { recursive: true, force: true }); } catch (_) {}
     }
     if (!poolIsExternal && pool) await driver.close(pool);
+    if (onStop) opts.signal.removeEventListener('abort', onStop);
   }
 }
 
+/** The error a stopped movement ends with: code 'STOPPED', so a caller can tell it from a failure. */
+function stoppedError() {
+  var err = new Error('Stopped by request');
+  err.code = 'STOPPED';
+  return err;
+}
+
 // Rollback a movement — drop from main + error tables, clear meta row.
 // Graceful queue abort first (in-flight SQLs allowed to settle).
 //
@@ -314,6 +369,11 @@ function normalizeOptions(opts) {
   opts.batchDir           = opts.batchDir           ||
     (process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions'));
   opts.keepBatchFiles     = !!opts.keepBatchFiles;
+  opts.staged             = !!opts.staged;
+  opts.writeTable         = null;
+  if (opts.staged && typeof opts.driver.mergeStaged !== 'function') {
+    throw new Error('upload: staged loads are not supported for this database — its driver has no mergeStaged');
+  }
   return opts;
 }
 
@@ -322,7 +382,7 @@ function safeBatchSize(columnCount, maxParams) {
   return Math.max(1, Math.floor(maxParams / paramsPerRow));
 }
 
-async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
+async function runBootstrap(driver, pool, targetTable, columns, primaryKeys, staged) {
   await driver.query(pool, driver.buildCreateTableSql(targetTable, columns, primaryKeys));
   await driver.query(pool, driver.buildCreateErrorTableSql(targetTable));
 
@@ -330,7 +390,9 @@ async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
   // per dialect (PG: IF NOT EXISTS; MySQL: introspect-then-DDL; MSSQL:
   // IF OBJECT_ID/COL_LENGTH), so the driver owns it, not this file.
   await driver.ensureMovementColumn(pool, targetTable);
-  await driver.ensureUpsertIndex(pool, targetTable, primaryKeys);
+  // A staged upsert matches keys in its move and needs no index — see the
+  // DuckDB driver's "staged loads" section for why it must not have one.
+  if (!staged) await driver.ensureUpsertIndex(pool, targetTable, primaryKeys);
 
   var targetSchema = await driver.getTableSchema(pool, targetTable);
   var { reconcileColumns } = require('./reconcile');
@@ -374,9 +436,11 @@ function watchWrites(opts, build, everyMs) {
 
 async function enqueueBatch(opts, rows, columns, batchRef) {
   var driver     = opts.driver;
-  var target     = opts.targetTable;
+  // Staged: plain inserts into the staging table; the keys are matched once,
+  // in the move.
+  var target     = opts.writeTable || opts.targetTable;
   var movementId = opts.movementId;
-  var pks        = opts.primaryKeys;
+  var pks        = opts.staged ? null : opts.primaryKeys;
 
   function rowsToSql(rs) { return driver.buildInsertSql(target, rs, columns, movementId, pks); }
 
@@ -401,12 +465,34 @@ function looksLikePool(v) {
   return v && typeof v.query === 'function' && typeof v.host !== 'string';
 }
 
+/**
+ * Did a staged movement reach its target? Read from the target's own ledger,
+ * written in the same transaction as the move — so for a run that ended
+ * without saying (its process was killed) this is the answer, not a guess.
+ * Returns the ledger row, or null.
+ */
+async function wasCommitted(opts) {
+  if (!opts || !opts.movementId) throw new Error('wasCommitted: movementId is required');
+  if (!opts.driver || typeof opts.driver.committedMovement !== 'function') {
+    throw new Error('wasCommitted: this database keeps no load ledger');
+  }
+  var poolIsExternal = looksLikePool(opts.connection);
+  var pool = poolIsExternal ? opts.connection : await opts.driver.connect(opts.connection);
+  try {
+    return await opts.driver.committedMovement(pool, opts.movementId);
+  } finally {
+    if (!poolIsExternal) await opts.driver.close(pool);
+  }
+}
+
 var { inferColumnType } = require('./infer');
 
 module.exports = {
   upload:          upload,
   watchWrites:     watchWrites,
+  stoppedError:    stoppedError,
   rollback:        rollback,
+  wasCommitted:    wasCommitted,
   inferColumns:    inferColumns,
   inferColumnType: inferColumnType,
   makeMetaStore:   require('./meta-store-knex')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "1.1.14",
+  "version": "1.1.16",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "files": [