@xeplr/actions 2.2.5 → 2.2.6

package/README.md

@@ -569,6 +569,8 @@ On Postgres, MySQL and SQL Server the staging table is shaped like the target (`
 
 Beside the tables it loads, the uploader makes three of its own: `<table>_import_errors` (refused rows), `<table>__xeplr_staging` (a staged load in progress) and, in a DuckDB file, `__xeplr_loads` (its ledger). `isInternalTable(name)` (also accepts a listing row `{ name }`) says whether a table is one of these, so an app listing a database's tables for people to pick from can leave them out. They still exist and are read where they belong.
 
+It also adds its own columns to every table it loads: `__xeplr_id__` (Postgres, MySQL, SQL Server) and `__xeplr_movement_id__` (every target), the row's id and the load that wrote it. `isInternalColumn(name)` (or `{ name }`) says whether a column is one of these. **They are never carried into another table:** when a loaded table is loaded somewhere else (an import copied into a cube's warehouse, a move between databases), the uploader drops them from the target's columns, whether inferred from the rows or declared by the caller, and the new load stamps its own. Carried along, they clashed with the target's own column (`Column with name __xeplr_movement_id__ already exists`) and would have marked rows with a load that never wrote them.
+
 ### A written DuckDB file keeps no index
 
 DuckDB 1.5.4–1.5.6 has a bug ([duckdb/duckdb#26106](https://github.com/duckdb/duckdb/issues/26106), open upstream): when a process dies with rows still only in the WAL, the next open replays them into the table and the next checkpoint drops them from every index on it. The first `DELETE` that reaches one fails with *"Failed to delete all rows from index"*, which is **FATAL**: the whole file is unusable until the process restarts. Any index, unique or not. A table with no index is unaffected.

package/index.js

@@ -48,6 +48,7 @@ module.exports = {
   uploader:      uploader,
   // The uploader's own tables (_import_errors, __xeplr_staging, __xeplr_loads), to leave out of table pickers.
   isInternalTable: require('./lib/drivers/db/internalTables').isInternalTable,
+  isInternalColumn: require('./lib/drivers/db/internalTables').isInternalColumn,
   inferColumns:  uploader.inferColumns,
   streaming:     streaming,
   drivers:       { db: dbDrivers, file: fileDrivers },

package/lib/drivers/db/internalTables.js

@@ -13,6 +13,13 @@ var ERRORS_SUFFIX = '_import_errors';
 var STAGING_SUFFIX = require('./staging').SUFFIX;
 var LOADS_LEDGER = '__xeplr_loads';
 
+// THE COLUMNS IT ADDS TO EVERY TABLE IT LOADS: a row's own id and the
+// movement that wrote it. They describe a load into THAT table, so they are
+// never carried into another one — loading a loaded table elsewhere (an
+// import copied into a cube's warehouse, a move between databases) leaves
+// them behind and the new load stamps its own (uploader/index.js).
+var INTERNAL_COLUMNS = ['__xeplr_id__', '__xeplr_movement_id__'];
+
 function endsWith(s, suffix) {
   return s.length > suffix.length && s.slice(-suffix.length) === suffix;
 }
@@ -23,8 +30,16 @@ function isInternalTable(name) {
   return endsWith(s, ERRORS_SUFFIX) || endsWith(s, STAGING_SUFFIX) || s === LOADS_LEDGER;
 }
 
+/** Whether a column is one the uploader adds itself. A name, or { name }. */
+function isInternalColumn(column) {
+  var s = String(column == null ? '' : (typeof column === 'object' ? column.name || '' : column));
+  return INTERNAL_COLUMNS.indexOf(s) !== -1;
+}
+
 module.exports = {
   isInternalTable: isInternalTable,
+  isInternalColumn: isInternalColumn,
+  INTERNAL_COLUMNS: INTERNAL_COLUMNS,
   ERRORS_SUFFIX: ERRORS_SUFFIX,
   STAGING_SUFFIX: STAGING_SUFFIX,
   LOADS_LEDGER: LOADS_LEDGER

package/lib/uploader/index.js

@@ -59,6 +59,7 @@
 //
 // See project_xeplr_actions_streaming.md for the full contract.
 
+var { isInternalColumn } = require('../drivers/db/internalTables');
 var path = require('path');
 var fsp = require('fs/promises');
 
@@ -149,6 +150,12 @@ async function upload(opts) {
             var scanLimit = Math.min(sample.length, opts.firstBatchScanRows);
             declared = inferColumns(sample.slice(0, scanLimit));
           }
+          // OUR OWN COLUMNS NEVER COME ALONG. A source that was itself loaded
+          // by the uploader (an import's table, a replica) has its own
+          // __xeplr_id__ and __xeplr_movement_id__; carried here they clashed
+          // with this table's ("already exists") and would have claimed rows
+          // for a load that never wrote them. This load stamps its own.
+          declared = declared.filter(function(c) { return !isInternalColumn(c); });
           columns = await runBootstrap(driver, pool, opts.targetTable, declared, opts.primaryKeys, opts.staged);
           if (opts.staged) {
             opts.writeTable = await driver.prepareStaging(pool, opts.targetTable);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xeplr/actions",
-  "version": "2.2.5",
+  "version": "2.2.6",
   "description": "Named-action registry with typed inputs and safe invocation — plus built-in actions for HTTP, DB, files, email, spawn",
   "main": "index.js",
   "scripts": {