@xeplr/actions 1.1.1

package/lib/uploader/index.js

@@ -0,0 +1,298 @@
+// Uploader — spool + SqlQueue + driver orchestrator.
+//
+//   const { upload, rollback } = require('@xeplr/actions').uploader;
+//
+//   await upload({
+//     source,          // async iterable of rows
+//     driver,          // e.g. require('.../drivers/db/postgres')
+//     connection,      // decrypted config object OR pool
+//     targetTable,
+//     primaryKeys,     // optional — enables UPSERT
+//     movementId,      // required — the correlation key
+//     queue,           // required — a SqlQueue instance
+//     connectionName,  // key used in queue.connections; defaults to targetTable
+//     batchSize:          5000,
+//     firstBatchScanRows: 1000,
+//     metaStore,       // defaults to meta-store-noop
+//     mtId1..mtId4,    // optional — forwarded to metaStore verbatim, tenant scoping
+//     details,         // optional — forwarded to metaStore verbatim, app-specific context
+//                       // (filename, connectionId, who ran it, ...) the engine itself
+//                       // has no business knowing about
+//     errorTable,      // defaults to `<targetTable>_import_errors`
+//     batchDir,        // where NDJSON files spool to. Defaults to os.tmpdir()/xeplr-actions
+//     keepBatchFiles,  // default false — clean up NDJSON files after success
+//   })
+//
+// EXECUTION MODEL (production-critical — read this):
+//
+//   The source (Excel, CSV, DB cursor, etc.) is decoupled from the target
+//   DB via the streaming/spool primitive. Rows flow like this:
+//
+//     [source rows] → spool writes NDJSON files → onBatch reads and pushes to SqlQueue → DB
+//
+//   Why: a slow DB (or a big SqlQueue backlog) must NEVER back-pressure all
+//   the way to the source parser. Some source parsers (xlsx-stream-reader
+//   specifically) emit rows from a SAX event handler and don't respect
+//   downstream backpressure — an in-memory row pile grows unboundedly until
+//   V8 OOMs. Spool breaks that chain by landing rows on disk between the
+//   two stages.
+//
+// See project_xeplr_actions_streaming.md for the full contract.
+
+var os = require('os');
+var path = require('path');
+var fsp = require('fs/promises');
+
+var { inferColumns } = require('./infer');
+var { spool, readBatchFile } = require('../streaming');
+
+// Postgres protocol caps at 65,535 parameters per query. We stay comfortably
+// under it — the effective batch cap becomes `maxParamsPerBatch / paramsPerRow`.
+var DEFAULT_MAX_PARAMS = 30000;
+
+async function upload(opts) {
+  opts = normalizeOptions(opts);
+  var driver = opts.driver;
+  var startedAt = Date.now();
+
+  var poolIsExternal = looksLikePool(opts.connection);
+  var pool = poolIsExternal ? opts.connection : await driver.connect(opts.connection);
+
+  // Shared state across the spool hooks (beforeAll → onBatch* → afterAll).
+  var columns = null;
+  var effectiveBatchSize = null;
+  var totalBatches = 0;
+
+  var spoolResult = null;
+  var runDir = null;
+
+  try {
+    var summary = await spool(opts.source, {
+      runId:        opts.movementId,
+      batchDir:     opts.batchDir,
+      batchSize:    opts.batchSize,
+      maxInFlight:  1,
+      maxMemoryMB:  opts.spoolMaxMemoryMB || 128,
+
+      target: {
+        // Bootstrap: first batch is on disk. Infer types from it, run DDL,
+        // reconcile against the target's existing schema, then compute the
+        // safe batch size for INSERT.
+        beforeAll: async function(ctx) {
+          runDir = ctx.batchDir;
+          var sample = await readBatchFile(ctx.firstBatchPath);
+          var scanLimit = Math.min(sample.length, opts.firstBatchScanRows);
+          var inferred = inferColumns(sample.slice(0, scanLimit));
+          columns = await runBootstrap(driver, pool, opts.targetTable, inferred, opts.primaryKeys);
+          effectiveBatchSize = Math.min(opts.batchSize, safeBatchSize(columns.length, opts.maxParamsPerBatch));
+          await opts.metaStore.recordStart(opts.movementId, {
+            targetTable:   opts.targetTable,
+            connectionKey: opts.connectionName,   // opaque label, not a real connection
+            dbType:        opts.dbType,            // recorded when the caller supplies it
+            columns:       columns,
+            primaryKeys:   opts.primaryKeys,
+            mtId1: opts.mtId1, mtId2: opts.mtId2, mtId3: opts.mtId3, mtId4: opts.mtId4,
+            details:       opts.details
+          });
+        },
+
+        // Each rotated NDJSON file becomes onBatch(rows, meta). We subdivide
+        // into safe-INSERT-sized chunks and enqueue each to SqlQueue.
+        // addToQueue awaits when queue is over its 200MB backpressure, which
+        // in turn awaits spool's next batch — the whole pipeline throttles
+        // correctly, but disk (not memory) is the buffer.
+        onBatch: async function(rows, _meta) {
+          if (!columns) throw new Error('uploader: onBatch before beforeAll — spool contract violated');
+          for (var off = 0; off < rows.length; off += effectiveBatchSize) {
+            var chunk = rows.slice(off, off + effectiveBatchSize);
+            await enqueueBatch(opts, chunk, columns);
+            totalBatches++;
+          }
+        }
+      }
+    });
+
+    spoolResult = summary;
+
+    // Empty source — spool created no batches, so beforeAll never fired.
+    // Nothing to do downstream; return an empty-but-valid summary.
+    if (!columns) {
+      // beforeAll never fired, so recordStart never happened — this is the
+      // only chance to give the row its plan/tenant context (meta-store-knex
+      // handles this via its insert-fallback path).
+      await opts.metaStore.recordEnd(opts.movementId, {
+        status: 'completed', totalRows: 0, totalBatches: 0, completed: 0, dropped: 0,
+        targetTable: opts.targetTable, connectionKey: opts.connectionName, dbType: opts.dbType,
+        mtId1: opts.mtId1, mtId2: opts.mtId2, mtId3: opts.mtId3, mtId4: opts.mtId4,
+        details: opts.details
+      });
+      return {
+        movementId:   opts.movementId,
+        tables:       { main: opts.targetTable, errors: opts.errorTable },
+        columns:      [],
+        totalRows:    0,
+        totalBatches: 0,
+        completed:    0,
+        dropped:      0,
+        aborted:      false,
+        durationMs:   Date.now() - startedAt
+      };
+    }
+
+    // Wait for the DB writes to finish (spool has finished feeding the queue).
+    await opts.queue.drain();
+
+    var stats = opts.queue.stats(opts.movementId) || {};
+    var status = stats.aborted ? 'aborted' : 'completed';
+    await opts.metaStore.recordEnd(opts.movementId, {
+      status:       status,
+      totalRows:    summary.totalRows,
+      totalBatches: totalBatches,
+      completed:    stats.completed || 0,
+      dropped:      stats.dropped || 0
+    });
+
+    return {
+      movementId:   opts.movementId,
+      tables:       { main: opts.targetTable, errors: opts.errorTable },
+      columns:      columns,
+      totalRows:    summary.totalRows,
+      totalBatches: totalBatches,
+      completed:    stats.completed || 0,
+      dropped:      stats.dropped || 0,
+      aborted:      !!stats.aborted,
+      durationMs:   Date.now() - startedAt
+    };
+  } finally {
+    // Best-effort cleanup: delete the NDJSON spool directory unless caller
+    // asked to keep it (e.g. for later resume / audit).
+    if (runDir && !opts.keepBatchFiles) {
+      try { await fsp.rm(runDir, { recursive: true, force: true }); } catch (_) {}
+    }
+    if (!poolIsExternal && pool) await driver.close(pool);
+  }
+}
+
+// Rollback a movement — drop from main + error tables, clear meta row.
+// Graceful queue abort first (in-flight SQLs allowed to settle).
+async function rollback(opts) {
+  if (!opts || !opts.movementId)  throw new Error('rollback: movementId is required');
+  if (!opts.driver)               throw new Error('rollback: driver is required');
+  if (!opts.connection)           throw new Error('rollback: connection is required');
+  if (!opts.targetTable)          throw new Error('rollback: targetTable is required');
+
+  if (opts.queue) {
+    opts.queue.abort(opts.movementId, 'rollback');
+    await opts.queue.drain();
+  }
+
+  var poolIsExternal = looksLikePool(opts.connection);
+  var pool = poolIsExternal ? opts.connection : await opts.driver.connect(opts.connection);
+
+  try {
+    var result = await opts.driver.rollbackMovement(pool, opts.targetTable, opts.movementId);
+    if (opts.metaStore) {
+      await opts.metaStore.recordEnd(opts.movementId, {
+        status: 'rolled-back',
+        mainDeleted:  result.mainDeleted,
+        errorDeleted: result.errorDeleted
+      });
+    }
+    return result;
+  } finally {
+    if (!poolIsExternal) await opts.driver.close(pool);
+  }
+}
+
+// ─── helpers ────────────────────────────────────────────────────────────
+
+function normalizeOptions(opts) {
+  opts = Object.assign({}, opts);
+  if (!opts.source)       throw new Error('upload: source is required');
+  if (!opts.driver)       throw new Error('upload: driver is required');
+  if (!opts.connection)   throw new Error('upload: connection is required');
+  if (!opts.targetTable)  throw new Error('upload: targetTable is required');
+  if (!opts.movementId)   throw new Error('upload: movementId is required');
+  if (!opts.queue)        throw new Error('upload: queue is required (a SqlQueue)');
+  opts.batchSize          = opts.batchSize          || 5000;
+  opts.firstBatchScanRows = opts.firstBatchScanRows || 1000;
+  // Per-dialect parameter ceiling: PG/MySQL ~65k, but SQL Server caps at 2100
+  // params + 1000 rows/statement. Drivers advertise `maxParams`; fall back to
+  // the PG-safe default when a driver doesn't.
+  opts.maxParamsPerBatch  = opts.maxParamsPerBatch  ||
+    (opts.driver && opts.driver.maxParams) || DEFAULT_MAX_PARAMS;
+  opts.metaStore          = opts.metaStore          || require('./meta-store-noop');
+  opts.errorTable         = opts.errorTable         || (opts.targetTable + '_import_errors');
+  opts.connectionName     = opts.connectionName     || opts.targetTable;
+  opts.batchDir           = opts.batchDir           ||
+    (process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions'));
+  opts.keepBatchFiles     = !!opts.keepBatchFiles;
+  return opts;
+}
+
+function safeBatchSize(columnCount, maxParams) {
+  var paramsPerRow = columnCount + 1;   // +1 for __xeplr_movement_id__
+  return Math.max(1, Math.floor(maxParams / paramsPerRow));
+}
+
+async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
+  await driver.query(pool, driver.buildCreateTableSql(targetTable, columns, primaryKeys));
+  await driver.query(pool, driver.buildCreateErrorTableSql(targetTable));
+
+  // Idempotent DDL via the driver's ensure-methods — the guard syntax differs
+  // per dialect (PG: IF NOT EXISTS; MySQL: introspect-then-DDL; MSSQL:
+  // IF OBJECT_ID/COL_LENGTH), so the driver owns it, not this file.
+  await driver.ensureMovementColumn(pool, targetTable);
+  await driver.ensureUpsertIndex(pool, targetTable, primaryKeys);
+
+  var targetSchema = await driver.getTableSchema(pool, targetTable);
+  var { reconcileColumns } = require('./reconcile');
+  var reconciled = reconcileColumns(columns, targetSchema, driver.dataTypeToLogical);
+
+  var missing = reconciled.filter(function(c) { return c.missing; });
+  if (missing.length > 0) {
+    throw new Error(
+      'upload: source has columns not present in target table "' + targetTable + '": ' +
+      missing.map(function(c) { return c.name; }).join(', ') +
+      '. ALTER TABLE to add them, or omit them from the source.'
+    );
+  }
+  return reconciled;
+}
+
+async function enqueueBatch(opts, rows, columns) {
+  var driver     = opts.driver;
+  var target     = opts.targetTable;
+  var movementId = opts.movementId;
+  var pks        = opts.primaryKeys;
+
+  function rowsToSql(rs) { return driver.buildInsertSql(target, rs, columns, movementId, pks); }
+
+  var built = rowsToSql(rows);
+  await opts.queue.addToQueue({
+    movementId: movementId,
+    connection: opts.connectionName,
+    sql:        built.sql,
+    params:     built.params,
+    meta: {
+      rows:       rows,
+      rowCount:   rows.length,
+      rowsToSql:  rowsToSql,
+      errorTable: opts.errorTable
+    }
+  });
+}
+
+function looksLikePool(v) {
+  return v && typeof v.query === 'function' && typeof v.host !== 'string';
+}
+
+var { inferColumnType } = require('./infer');
+
+module.exports = {
+  upload:          upload,
+  rollback:        rollback,
+  inferColumns:    inferColumns,
+  inferColumnType: inferColumnType,
+  makeMetaStore:   require('./meta-store-knex')
+};

package/lib/uploader/infer.js

@@ -0,0 +1,76 @@
+// Type inference from the first N rows.
+//
+// Precedence when a column has mixed values:
+//   1. Any non-null value is an Array          → 'array'
+//   2. Any non-null value is a plain object    → 'object'
+//   3. All non-null values are Date instances OR ISO-parseable strings → 'datetime'
+//   4. All non-null values are booleans        → 'boolean'
+//   5. All non-null values are numbers         → 'number'
+//   6. Anything else / all null                → 'string'
+//
+// Text always wins ties (per user preference: "string always wins... no complexity").
+
+var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
+
+function inferColumnType(values) {
+  var seenObject = false, seenArray = false;
+  var allBool = true, sawBool = false;
+  var allNum  = true, sawNum  = false;
+  var allDate = true, sawDate = false;
+  var allNull = true;
+
+  for (var i = 0; i < values.length; i++) {
+    var v = values[i];
+    if (v === undefined || v === null) continue;
+    allNull = false;
+
+    if (Array.isArray(v))                                    { seenArray = true;  allBool = allNum = allDate = false; continue; }
+    if (v instanceof Date)                                   { sawDate = true; allBool = allNum = false; continue; }
+    if (typeof v === 'boolean')                              { sawBool = true; allNum = allDate = false; continue; }
+    if (typeof v === 'number' && !isNaN(v))                  { sawNum = true;  allBool = allDate = false; continue; }
+    if (typeof v === 'object')                               { seenObject = true; allBool = allNum = allDate = false; continue; }
+    if (typeof v === 'string') {
+      if (ISO_DATE_RE.test(v)) { sawDate = true; allBool = allNum = false; continue; }
+      allBool = allNum = allDate = false;
+      continue;
+    }
+    // Anything else falls through to string.
+    allBool = allNum = allDate = false;
+  }
+
+  if (seenArray)  return 'array';
+  if (seenObject) return 'object';
+  if (sawDate && allDate) return 'datetime';
+  if (sawBool && allBool) return 'boolean';
+  if (sawNum  && allNum)  return 'number';
+  return 'string';
+}
+
+// Given a buffer of rows, return an ordered list of column defs:
+//   [{ name, type }]
+// Column order follows the FIRST row's keys, then any additional keys
+// encountered in later rows appended in insertion order.
+function inferColumns(rows) {
+  if (!rows || rows.length === 0) return [];
+
+  var order = [];
+  var seen = new Set();
+  var valuesByCol = {};
+
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    if (!row || typeof row !== 'object') continue;
+    var keys = Object.keys(row);
+    for (var j = 0; j < keys.length; j++) {
+      var k = keys[j];
+      if (!seen.has(k)) { seen.add(k); order.push(k); valuesByCol[k] = []; }
+      valuesByCol[k].push(row[k]);
+    }
+  }
+
+  return order.map(function(name) {
+    return { name: name, type: inferColumnType(valuesByCol[name]) };
+  });
+}
+
+module.exports = { inferColumns: inferColumns, inferColumnType: inferColumnType };

package/lib/uploader/meta-store-knex.js

@@ -0,0 +1,136 @@
+// Knex-backed MetaStore — persists movement metadata to the `import_meta`
+// table in the framework's xeplr_configs DB (control plane), shared across
+// apps. Implements the MetaStore interface the uploader calls: getLast /
+// recordStart / recordProgress / recordEnd.
+//
+// Peer deps: knex (+ the pg client — bootstrapConfigDb/import_meta are
+// Postgres-only, see xeplr-db/lib/bootstrap-config.js).
+//
+// Factory shape — bind the knex handle AND this app's name at configuration
+// time (service is deliberately constructor-supplied, not env-derived — the
+// app that's calling knows its own name, no reason to make callers thread it
+// through every recordStart/recordEnd call):
+//   const makeStore = require('@xeplr/actions/lib/uploader/meta-store-knex');
+//   const store = makeStore(knex, { table: 'import_meta', service: 'xeplr-bi' });
+//   await upload({ ..., metaStore: store });
+//
+// Records NO secrets: `connection_key` is an opaque label the consumer chose,
+// never a real connection. `columns`/`primary_keys` describe the movement
+// plan. `details` (app-specific context — filename, connectionId, who ran
+// it) and `mtId1`-`mtId4` (tenant scoping, same column names as BaseModel)
+// are set once at recordStart and never touched by recordEnd — this table is
+// shared across apps, so every read/write is scoped by `service` too.
+
+module.exports = function makeKnexMetaStore(knex, options) {
+  if (!knex) throw new Error('meta-store-knex: a knex instance is required');
+  if (!options || !options.service) throw new Error('meta-store-knex: options.service is required');
+  var TABLE = options.table || 'import_meta';
+  var SERVICE = options.service;
+
+  return {
+    // Most-recent movement for a target table (resume / dedupe / audit).
+    // Scoped to this store's own service — a movement recorded by another
+    // app is never "the last one" from this app's point of view.
+    async getLast(opts) {
+      var targetTable = opts && opts.targetTable;
+      if (!targetTable) return null;
+      var row = await knex(TABLE)
+        .where({ service: SERVICE, target_table: targetTable })
+        .orderBy('started_at', 'desc')
+        .first();
+      return row || null;   // pg returns jsonb columns already parsed
+    },
+
+    // Open the movement row in 'running' state. Upsert on id so a re-run /
+    // resume of the same movementId overwrites its prior 'running' row.
+    async recordStart(movementId, plan) {
+      plan = plan || {};
+      await knex(TABLE)
+        .insert({
+          id:             movementId,
+          service:        SERVICE,
+          mtId1:          plan.mtId1 || null,
+          mtId2:          plan.mtId2 || null,
+          mtId3:          plan.mtId3 || null,
+          mtId4:          plan.mtId4 || null,
+          details:        plan.details ? JSON.stringify(plan.details) : null,
+          target_table:   plan.targetTable   || null,
+          connection_key: plan.connectionKey || null,
+          db_type:        plan.dbType         || null,
+          status:         'running',
+          primary_keys:   plan.primaryKeys ? JSON.stringify(plan.primaryKeys) : null,
+          columns:        plan.columns     ? JSON.stringify(plan.columns)     : null,
+          error:          null,
+          started_at:         knex.fn.now(),
+          ended_at:           null,
+          recordModifiedDate: knex.fn.now()
+        })
+        .onConflict('id')
+        .merge();
+    },
+
+    // Increment running counters. delta = { completed?, dropped?, totalRows?, totalBatches? }
+    async recordProgress(movementId, delta) {
+      delta = delta || {};
+      var patch = { recordModifiedDate: knex.fn.now() };
+      var COLS = { completed: 'completed', dropped: 'dropped', totalRows: 'total_rows', totalBatches: 'total_batches' };
+      var touched = false;
+      for (var key in COLS) {
+        if (typeof delta[key] === 'number') {
+          var col = COLS[key];
+          patch[col] = knex.raw('?? + ?', [col, delta[key]]);
+          touched = true;
+        }
+      }
+      if (!touched) return;
+      await knex(TABLE).where({ id: movementId }).update(patch);
+    },
+
+    // Close the movement. Upserts: the empty-source path calls recordEnd
+    // WITHOUT a prior recordStart (beforeAll never fires), so a missing row
+    // must be created rather than silently no-op'd.
+    async recordEnd(movementId, result) {
+      result = result || {};
+      var patch = {
+        status:             result.status || 'completed',
+        ended_at:           knex.fn.now(),
+        recordModifiedDate: knex.fn.now()
+      };
+      if (typeof result.totalRows    === 'number') patch.total_rows    = result.totalRows;
+      if (typeof result.totalBatches === 'number') patch.total_batches = result.totalBatches;
+      if (typeof result.completed    === 'number') patch.completed     = result.completed;
+      if (typeof result.dropped      === 'number') patch.dropped       = result.dropped;
+      if (result.error) patch.error = String(result.error.message || result.error);
+      // rollback reports mainDeleted/errorDeleted — keep them in meta jsonb.
+      if (result.mainDeleted != null || result.errorDeleted != null) {
+        patch.meta = JSON.stringify({ mainDeleted: result.mainDeleted, errorDeleted: result.errorDeleted });
+      }
+
+      var updated = await knex(TABLE).where({ id: movementId }).update(patch);
+      if (!updated) {
+        // No prior recordStart (empty-source path — beforeAll never fires) —
+        // this is the only chance to stamp service/tenant/plan context, or
+        // the row is left sparse forever. Caller may pass these through
+        // result for exactly this case (see uploader/index.js's empty-source
+        // recordEnd call).
+        var fallback = {
+          id: movementId,
+          service: SERVICE,
+          mtId1: result.mtId1 || null,
+          mtId2: result.mtId2 || null,
+          mtId3: result.mtId3 || null,
+          mtId4: result.mtId4 || null,
+          details: result.details ? JSON.stringify(result.details) : null,
+          target_table: result.targetTable || null,
+          connection_key: result.connectionKey || null,
+          db_type: result.dbType || null,
+          started_at: knex.fn.now()
+        };
+        await knex(TABLE)
+          .insert(Object.assign(fallback, patch))
+          .onConflict('id')
+          .merge();
+      }
+    }
+  };
+};

package/lib/uploader/meta-store-noop.js

@@ -0,0 +1,11 @@
+// Default MetaStore — records nothing, returns nothing.
+// Used when the uploader is called without a real metaStore instance.
+// Consumers wire in meta-store-knex.js (optional peer: knex + pg) or
+// their own implementation.
+
+module.exports = {
+  async getLast(_opts) { return null; },
+  async recordStart(_uploadId, _plan) { /* no-op */ },
+  async recordProgress(_uploadId, _delta) { /* no-op */ },
+  async recordEnd(_uploadId, _result) { /* no-op */ }
+};

package/lib/uploader/reconcile.js

@@ -0,0 +1,88 @@
+// Reconcile source-inferred column types with an existing target table's
+// actual schema. The target table is authoritative — its column types WIN.
+// Source values will be coerced to match the target's type at INSERT.
+//
+// Special case: TEXT / VARCHAR target columns accept anything, so any
+// source type widens safely to 'string' (numbers/dates/bools/objects all
+// get serialized on INSERT).
+//
+// Columns that exist in the source but NOT in the target are surfaced as
+// `missing: true` — the caller (uploader) decides whether to ALTER TABLE
+// ADD or reject the upload.
+//
+// Framework-managed columns (__xeplr_id__, __xeplr_movement_id__) are
+// ignored during reconciliation.
+
+var INTERNAL_COLS = new Set(['__xeplr_id__', '__xeplr_movement_id__']);
+
+// Postgres data_type strings → our logical types.
+var PG_TO_LOGICAL = {
+  'text':                        'string',
+  'character varying':           'string',
+  'character':                   'string',
+  'uuid':                        'string',
+  'integer':                     'number',
+  'bigint':                      'number',
+  'smallint':                    'number',
+  'numeric':                     'number',
+  'real':                        'number',
+  'double precision':            'number',
+  'boolean':                     'boolean',
+  'timestamp with time zone':    'datetime',
+  'timestamp without time zone': 'datetime',
+  'date':                        'datetime',
+  'time with time zone':         'datetime',
+  'time without time zone':      'datetime',
+  'json':                        'object',
+  'jsonb':                       'object',
+  'ARRAY':                       'array'
+};
+
+function pgDataTypeToLogical(dataType) {
+  return PG_TO_LOGICAL[dataType] || 'string';   // unknown → assume TEXT-like
+}
+
+/**
+ * @param {Array<{name,type}>}                                   sourceColumns
+ * @param {Array<{name,dataType,udtName?}> | null | undefined}   targetSchema
+ * @param {(dataType:string)=>string} [toLogical]  Dialect-specific native
+ *   data_type → logical type mapper. Defaults to the Postgres mapping so
+ *   existing callers are unaffected; the uploader passes driver.dataTypeToLogical
+ *   so MySQL/MSSQL target schemas reconcile against their own type strings.
+ * @returns {Array<{ name, type, sourceType?, targetType?, missing? }>}
+ *   `type` is the EFFECTIVE type — what values will be coerced to before
+ *   INSERT. `sourceType` is preserved for auditing; `missing` flags columns
+ *   present in source but absent from target.
+ */
+function reconcileColumns(sourceColumns, targetSchema, toLogical) {
+  sourceColumns = sourceColumns || [];
+  toLogical = toLogical || pgDataTypeToLogical;
+
+  if (!targetSchema || targetSchema.length === 0) {
+    // No existing table → source types stand as effective.
+    return sourceColumns.map(function(s) { return { name: s.name, type: s.type }; });
+  }
+
+  var targetByName = {};
+  for (var i = 0; i < targetSchema.length; i++) {
+    var t = targetSchema[i];
+    if (INTERNAL_COLS.has(t.name)) continue;
+    targetByName[t.name] = toLogical(t.dataType);
+  }
+
+  return sourceColumns.map(function(src) {
+    if (INTERNAL_COLS.has(src.name)) return { name: src.name, type: src.type };
+    var targetType = targetByName[src.name];
+    if (targetType === undefined) {
+      return { name: src.name, type: src.type, sourceType: src.type, missing: true };
+    }
+    // Target's type wins.
+    if (targetType === src.type) return { name: src.name, type: src.type };
+    return { name: src.name, type: targetType, sourceType: src.type, targetType: targetType };
+  });
+}
+
+module.exports = {
+  reconcileColumns:    reconcileColumns,
+  pgDataTypeToLogical: pgDataTypeToLogical
+};