@xeplr/actions 1.1.15 → 1.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -4
- package/lib/builtins/db/move.js +14 -1
- package/lib/drivers/db/duckdb.js +132 -0
- package/lib/uploader/index.js +71 -5
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -72,7 +72,7 @@ var moved = await actions.runAction(actions.builtins.dbMove, {
|
|
|
72
72
|
| `ActionMissingDependencyError` | Thrown by `register()` for a missing `requires` module. |
|
|
73
73
|
| `TransientError` | For an action to signal "retry me". Nothing in this package (or `@xeplr/jobs`) acts on it yet. |
|
|
74
74
|
| `builtins` | The built-in action modules — see [Built-in actions](#built-in-actions). |
|
|
75
|
-
| `uploader` | `{ upload, rollback, inferColumns, inferColumnType, makeMetaStore }` — see [Uploader](#uploader). |
|
|
75
|
+
| `uploader` | `{ upload, rollback, wasCommitted, inferColumns, inferColumnType, makeMetaStore }` — see [Uploader](#uploader). |
|
|
76
76
|
| `inferColumns(rows)` | `[{ name, type }]` from sample rows. |
|
|
77
77
|
| `streaming` | `{ spool, resume, readBatchFile, checkpoint }` — rows to rotating NDJSON batch files with ordered hooks and resumable checkpoints. |
|
|
78
78
|
| `drivers.db` | `{ getDriver(type), checkDriverRequires, SUPPORTED }` — `postgres`, `mysql`, `mssql`, `mongo`, `duckdb`. |
|
|
@@ -141,6 +141,7 @@ The db actions `db-fetch`, `db-push` and `db-procedure` accept a connection two
|
|
|
141
141
|
| `writeMode` | `append` (default), `replace` (TRUNCATE the target first if it exists, DELETE if TRUNCATE fails), `upsert` (**requires** `primaryKeys`). |
|
|
142
142
|
| `window` | `{ column, from, to }` → `column >= from AND column < to` (a null edge is left out). `{ columns: [{ column, from }] }` → OR of `column >= from` (null `from`s left out). For a procedure: `{ from, to, fromParam, toParam }`, passed as parameters. A window with none of `column`, `columns`, `fromParam`, `toParam` is treated as no window. |
|
|
143
143
|
| `where` | Raw SQL condition, ANDed with the window in parentheses, run on the source. Not parameterised. Ignored for `mode: 'procedure'`. |
|
|
144
|
+
| `staged` | `true`: write into `<targetTable>__xeplr_staging` and move it into the target in one transaction at the end (see [Staged loads](#staged-loads)). DuckDB targets only. Default `false`. |
|
|
144
145
|
| `movementId` | Correlation key. Default `system.occurrenceId`, else `mv_<id>`. |
|
|
145
146
|
| `batchSize`, `concurrency` | Default 5000 and 4. |
|
|
146
147
|
|
|
@@ -185,9 +186,10 @@ await actions.uploader.upload({
|
|
|
185
186
|
batchSize: 5000, firstBatchScanRows: 1000,
|
|
186
187
|
metaStore, mtId1, mtId2, mtId3, mtId4, details,
|
|
187
188
|
errorTable, batchDir, keepBatchFiles, onProgress,
|
|
188
|
-
signal
|
|
189
|
+
signal, // optional AbortSignal: stop on request
|
|
190
|
+
staged // optional: see Staged loads
|
|
189
191
|
})
|
|
190
|
-
// → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, durationMs }
|
|
192
|
+
// → { movementId, tables: { main, errors }, columns, totalRows, totalBatches, completed, dropped, aborted, merged, durationMs }
|
|
191
193
|
```
|
|
192
194
|
|
|
193
195
|
`onProgress({ rowsRead, batches, rowsWritten, phase? })` fires after each batch is read, with `rowsWritten` from the write queue's own count, so a caller can show both and see whether writing keeps up. After the read, while the queue drains, it fires again each time the written count moves (`phase: 'writing'`), never on a bare timer, so a stuck write goes quiet.
|
|
@@ -209,6 +211,18 @@ Logical types and what each driver creates:
|
|
|
209
211
|
|
|
210
212
|
Inference (`inferColumns`): any array → `array`; any object → `object`; all `Date` or ISO-date strings → `datetime`; all booleans → `boolean`; all numbers → `number`; anything else, or all null → `string`.
|
|
211
213
|
|
|
214
|
+
### Staged loads
|
|
215
|
+
|
|
216
|
+
With `staged: true` a load **never writes into its target while it runs**. Batches land in `<targetTable>__xeplr_staging` (made empty, shaped like the target, at the start). When every batch is in, one transaction moves them into the target and writes a row to `__xeplr_loads` (`movement_id`, `target_table`, `inserted`, `replaced`, `committed_at`); then the staging table is dropped. `merged` in the result is `{ inserted, replaced }`; `onProgress` reports `phase: 'merging'` just before the move.
|
|
217
|
+
|
|
218
|
+
- **Stopped, failed, aborted or killed before the move**: the target is untouched. The staging table is dropped (or, after a crash, by the next load's start). Undoing it costs what the load wrote, never what the target holds; there is nothing to roll back.
|
|
219
|
+
- **Crash during the move**: the transaction is discarded on reopen and the target is as it was.
|
|
220
|
+
- **Did it land?** `uploader.wasCommitted({ driver, connection, movementId })` reads the ledger row, written in the same transaction as the move: a row means it landed, `null` means nothing did. No scan of the target.
|
|
221
|
+
- **Upsert** replaces target rows whose key is staged (`DELETE … USING` staging, then `INSERT`). Within one load the last row staged for a key wins, as under `ON CONFLICT DO UPDATE`; rows with a NULL key match nothing and are kept. **No unique index**: a staged load drops `<targetTable>_upsert_uniq` and never creates it. DuckDB's DELETE through an index that a killed load left out of step with its rows fails with "Failed to delete all rows from index", and that failure invalidates the whole database.
|
|
222
|
+
- **Cost** is one statement at the end. Measured with 50 lakh staged rows into 50 crore: append 0.3 s (independent of the target's size), upsert 0.4–7 s depending on how widely the replaced keys are spread.
|
|
223
|
+
|
|
224
|
+
A driver opts in by implementing `prepareStaging`, `mergeStaged`, `dropStaging` and `committedMovement`; `upload` refuses `staged` for one that does not.
|
|
225
|
+
|
|
212
226
|
`uploader.rollback({ movementId, driver, connection, targetTable, metaStore?, queue? })` aborts the movement's queued work, deletes its rows from the target and error tables by `__xeplr_movement_id__` / `movement_id`, and records `rolled-back`.
|
|
213
227
|
|
|
214
228
|
### Formats and file sources
|
|
@@ -290,6 +304,7 @@ await driver.query(pool, userSql, params) // reads only
|
|
|
290
304
|
| `db-fetch` streams to disk by default; `streaming_mode: false` only for small reads. | Bounded memory at both ends for cross-engine replication. |
|
|
291
305
|
| **Half-open windows** (`from <= x < to`), no +1 second. | Consecutive windows meet exactly; closed windows move the boundary row twice. |
|
|
292
306
|
| `db-move` `upsert` without `primaryKeys` is refused, not downgraded to append. | A silent append is a duplicate every run, found a week later. |
|
|
307
|
+
| A **staged** load touches its target only in one transaction at the end, and keeps no unique index. | Undoing a stopped load must cost what it wrote, not a rebuild of the table it was adding to; and a DELETE through a damaged DuckDB index invalidates the database. |
|
|
293
308
|
| `replace` truncates before reading; on failure the target stays empty. | Accepted trade against keeping a second copy of the data. |
|
|
294
309
|
| A procedure call with a window but no `fromParam`/`toParam` is refused (`db-procedure`, and the shared builder `db-move` uses). | A procedure cannot be filtered from outside; dropping the window would reprocess its whole history every run while reporting success. |
|
|
295
310
|
| `db-move` uses the source table's declared types for `mode: 'table'`. | Inference is lossy: Postgres `numeric` arrives as a string and would create a TEXT column; an all-null sample becomes text; `'007'` loses its zeros. |
|
|
@@ -316,7 +331,7 @@ node --test test/uploader/reconcile.test.js
|
|
|
316
331
|
| needs nothing | needs a database |
|
|
317
332
|
|---|---|
|
|
318
333
|
| `test/procedure-call.test.js`, `test/streaming/spool.test.js`, `test/uploader/reconcile.test.js`, `test/write-progress.test.js`, `test/stop-movement.test.js`, `test/formats/csv-gnarly.test.js`, `test/drivers/mysql.test.js`, `test/drivers/mssql.test.js`, `test/drivers/query-columns.test.js`, `test/builtins/email.test.js` (loads the package root, so `@xeplr/db` must be resolvable) | Postgres at `localhost:5435` (`PG_PASSWORD`, default `postgres`; database `xeplr_actions_test`): `test/drivers/postgres.test.js`, `test/builtins/db-push.test.js`, `test/builtins/file-upload.test.js`, `test/uploader/upload.test.js`; plus `@xeplr/db` for `test/uploader/meta-store-knex.test.js` |
|
|
319
|
-
| `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
|
|
334
|
+
| `test/drivers/duckdb.test.js`, `test/drivers/duckdb-values.test.js`, `test/staged-load.test.js` (needs `@duckdb/node-api`) | MySQL `localhost:3306` (`MYSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mysql-integration.test.js` |
|
|
320
335
|
| | SQL Server `localhost:1433` (`MSSQL_HOST/PORT/USER/PASSWORD`): `test/drivers/mssql-integration.test.js` |
|
|
321
336
|
| | Postgres + MySQL + SQL Server (`PG_*`, `MYSQL_*`, `MSSQL_*`): `test/builtins/db-replication.test.js` |
|
|
322
337
|
|
package/lib/builtins/db/move.js
CHANGED
|
@@ -94,6 +94,11 @@ module.exports = {
|
|
|
94
94
|
{ name: 'primaryKeys', type: 'array', order: 12, group: 'Write options',
|
|
95
95
|
description: 'Target column names for UPSERT. Required when writeMode="upsert".' },
|
|
96
96
|
|
|
97
|
+
{ name: 'staged', type: 'boolean', default: false, order: 12.5, group: 'Write options',
|
|
98
|
+
description: 'Write into a staging table and move it into the target in one transaction at ' +
|
|
99
|
+
'the end, so a movement that stops or fails midway never touches the target. ' +
|
|
100
|
+
'DuckDB targets only.' },
|
|
101
|
+
|
|
97
102
|
{ name: 'window', type: 'object', order: 13, group: 'Advanced',
|
|
98
103
|
description: 'One window: { column, from, to }. Half-open — from <= x < to. ' +
|
|
99
104
|
'Omitted = no filter. For mode="procedure", also needs { fromParam, toParam }.' },
|
|
@@ -164,6 +169,9 @@ module.exports = {
|
|
|
164
169
|
{ phase: 'start', event: 'columns', count: (input.columns || []).length });
|
|
165
170
|
system.log('Write mode: ' + describeWriteMode(writeMode, primaryKeys),
|
|
166
171
|
{ phase: 'start', event: 'write_mode', writeMode: writeMode, primaryKeys: primaryKeys || [] });
|
|
172
|
+
if (input.staged) system.log('Staged: rows collect in "' + input.targetTable + '__xeplr_staging" and move into "' +
|
|
173
|
+
input.targetTable + '" in one step at the end — a stop or failure before then leaves "' + input.targetTable + '" untouched',
|
|
174
|
+
{ phase: 'start', event: 'staged', stagingTable: input.targetTable + '__xeplr_staging' });
|
|
167
175
|
system.log('Window: ' + describeWindow(input.window),
|
|
168
176
|
{ phase: 'start', event: 'window',
|
|
169
177
|
// Bounds are CONFIGURATION — a date range somebody chose — not rows
|
|
@@ -306,7 +314,8 @@ module.exports = {
|
|
|
306
314
|
// has to survive being stored in a job row.
|
|
307
315
|
onProgress: system.onProgress || null,
|
|
308
316
|
// An AbortSignal from the caller: stop reading, drop the queued writes.
|
|
309
|
-
signal: system.signal || null
|
|
317
|
+
signal: system.signal || null,
|
|
318
|
+
staged: Boolean(input.staged)
|
|
310
319
|
});
|
|
311
320
|
|
|
312
321
|
// Every count upload() has, not just the headline one — "read" and
|
|
@@ -320,6 +329,10 @@ module.exports = {
|
|
|
320
329
|
bits.push(result.dropped + ' dropped' +
|
|
321
330
|
(droppedGroups ? ' in ' + droppedGroups + ' failed batch(es) — see the BATCH … FAILED lines for which' : ''));
|
|
322
331
|
}
|
|
332
|
+
if (result.merged) {
|
|
333
|
+
bits.push('moved into the table in one step (' + result.merged.inserted + ' in' +
|
|
334
|
+
(result.merged.replaced ? ', ' + result.merged.replaced + ' replaced' : '') + ')');
|
|
335
|
+
}
|
|
323
336
|
bits.push((result.totalBatches || 0) + ' batch' + (result.totalBatches === 1 ? '' : 'es'));
|
|
324
337
|
bits.push(((result.durationMs || 0) / 1000).toFixed(1) + 's');
|
|
325
338
|
// THE LINE THE WHOLE LOG EXISTS FOR. Every number a discrepancy
|
package/lib/drivers/db/duckdb.js
CHANGED
|
@@ -738,6 +738,131 @@ async function rollbackMovement(pool, tableName, movementId) {
|
|
|
738
738
|
return result;
|
|
739
739
|
}
|
|
740
740
|
|
|
741
|
+
// ── staged loads ─────────────────────────────────────────────────────────
|
|
742
|
+
//
|
|
743
|
+
// A STAGED load never writes into its target while it runs. Its batches land
|
|
744
|
+
// in `<target>__xeplr_staging`; when every batch is in, ONE transaction moves
|
|
745
|
+
// them into the target and records the load in `__xeplr_loads`. So a load
|
|
746
|
+
// that stops, fails or is killed midway has touched only the staging table:
|
|
747
|
+
// undoing it costs what IT wrote, never what the target already held, and no
|
|
748
|
+
// rollback of the target is ever needed.
|
|
749
|
+
//
|
|
750
|
+
// DuckDB commits a transaction whole or not at all, so a crash during the
|
|
751
|
+
// move leaves the target as it was. The ledger row is written in that same
|
|
752
|
+
// transaction: if it is there the load landed, if it is not nothing did —
|
|
753
|
+
// answered without scanning the target (see committedMovement).
|
|
754
|
+
//
|
|
755
|
+
// NO UNIQUE INDEX. A staged upsert matches keys in the move itself, so the
|
|
756
|
+
// upsert index is dropped rather than created. It was the index a copy killed
|
|
757
|
+
// mid-write left out of step with its rows, and a DELETE through such an index
|
|
758
|
+
// fails with "Failed to delete all rows from index" — fatally, invalidating
|
|
759
|
+
// the whole database.
|
|
760
|
+
|
|
761
|
+
var STAGING_SUFFIX = '__xeplr_staging';
|
|
762
|
+
var LOADS_LEDGER = '__xeplr_loads';
|
|
763
|
+
|
|
764
|
+
function stagingTableFor(tableName) { return tableName + STAGING_SUFFIX; }
|
|
765
|
+
|
|
766
|
+
/** Run fn on ONE connection — a transaction's statements must not be spread across the pool. */
|
|
767
|
+
async function withConnection(pool, fn) {
|
|
768
|
+
var entry = entryOf(pool);
|
|
769
|
+
var conn = await acquire(entry);
|
|
770
|
+
try {
|
|
771
|
+
return await fn(conn);
|
|
772
|
+
} finally {
|
|
773
|
+
release(entry, conn);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/**
|
|
778
|
+
* A fresh, empty staging table shaped exactly like the target (which must
|
|
779
|
+
* exist). A staging table already there is a load that died before its move,
|
|
780
|
+
* and is dropped: nothing in it ever reached the target.
|
|
781
|
+
*/
|
|
782
|
+
async function prepareStaging(pool, tableName) {
|
|
783
|
+
var staging = stagingTableFor(tableName);
|
|
784
|
+
await query(pool, 'DROP INDEX IF EXISTS ' + quoteIdent(tableName + '_upsert_uniq'));
|
|
785
|
+
await query(pool, 'DROP TABLE IF EXISTS ' + quoteIdent(staging));
|
|
786
|
+
await query(pool, 'CREATE TABLE ' + quoteIdent(staging) + ' AS SELECT * FROM ' + quoteIdent(tableName) + ' LIMIT 0');
|
|
787
|
+
return staging;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
async function dropStaging(pool, tableName) {
|
|
791
|
+
await query(pool, 'DROP TABLE IF EXISTS ' + quoteIdent(stagingTableFor(tableName)));
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
/**
|
|
795
|
+
* Move a load's staged rows into its target, in one transaction with its
|
|
796
|
+
* ledger row, then drop the staging table.
|
|
797
|
+
*
|
|
798
|
+
* With primaryKeys it is an upsert: target rows whose key is staged are
|
|
799
|
+
* replaced. Within one load the last row staged for a key wins, as it did
|
|
800
|
+
* under ON CONFLICT DO UPDATE; a row with a NULL key matches nothing, and is
|
|
801
|
+
* kept, as ON CONFLICT kept it.
|
|
802
|
+
*
|
|
803
|
+
* Returns { inserted, replaced }.
|
|
804
|
+
*/
|
|
805
|
+
async function mergeStaged(pool, opts) {
|
|
806
|
+
var target = quoteIdent(opts.targetTable);
|
|
807
|
+
var staging = quoteIdent(stagingTableFor(opts.targetTable));
|
|
808
|
+
var ledger = quoteIdent(LOADS_LEDGER);
|
|
809
|
+
var pks = (opts.primaryKeys && opts.primaryKeys.length) ? opts.primaryKeys : null;
|
|
810
|
+
var cols = [INTERNAL_MOVEMENT_COL].concat(opts.columns.map(function(c) { return c.name; }))
|
|
811
|
+
.map(quoteIdent).join(', ');
|
|
812
|
+
|
|
813
|
+
var source = staging;
|
|
814
|
+
if (pks) {
|
|
815
|
+
var keys = pks.map(quoteIdent);
|
|
816
|
+
source = '(SELECT ' + cols + ' FROM ' + staging +
|
|
817
|
+
' QUALIFY row_number() OVER (PARTITION BY ' + keys.join(', ') + ' ORDER BY rowid DESC) = 1' +
|
|
818
|
+
' OR ' + keys.map(function(k) { return k + ' IS NULL'; }).join(' OR ') + ')';
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
var counts = await withConnection(pool, async function(conn) {
|
|
822
|
+
await conn.run('CREATE TABLE IF NOT EXISTS ' + ledger +
|
|
823
|
+
' (movement_id VARCHAR, target_table VARCHAR, inserted BIGINT, replaced BIGINT, committed_at TIMESTAMPTZ)');
|
|
824
|
+
await conn.run('BEGIN TRANSACTION');
|
|
825
|
+
try {
|
|
826
|
+
var replaced = 0;
|
|
827
|
+
if (pks) {
|
|
828
|
+
var del = await conn.run('DELETE FROM ' + target + ' AS t USING ' + staging + ' AS s WHERE ' +
|
|
829
|
+
pks.map(function(k) { return 't.' + quoteIdent(k) + ' = s.' + quoteIdent(k); }).join(' AND '));
|
|
830
|
+
replaced = Number(del.rowsChanged) || 0;
|
|
831
|
+
}
|
|
832
|
+
var ins = await conn.run('INSERT INTO ' + target + ' (' + cols + ') SELECT ' + cols + ' FROM ' + source);
|
|
833
|
+
var inserted = Number(ins.rowsChanged) || 0;
|
|
834
|
+
await conn.run('INSERT INTO ' + ledger + ' VALUES ($1, $2, $3, $4, now())',
|
|
835
|
+
[opts.movementId, opts.targetTable, inserted, replaced]);
|
|
836
|
+
await conn.run('COMMIT');
|
|
837
|
+
return { inserted: inserted, replaced: replaced };
|
|
838
|
+
} catch (err) {
|
|
839
|
+
await conn.run('ROLLBACK').catch(function() {});
|
|
840
|
+
throw err;
|
|
841
|
+
}
|
|
842
|
+
});
|
|
843
|
+
|
|
844
|
+
// After the commit, and allowed to fail: the load has landed, and the next
|
|
845
|
+
// load drops a leftover staging table anyway.
|
|
846
|
+
await dropStaging(pool, opts.targetTable).catch(function() {});
|
|
847
|
+
return counts;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Did this movement's staged load reach its target? The ledger row, or null.
|
|
852
|
+
* The answer a run that ended without saying (a killed process) needs.
|
|
853
|
+
*/
|
|
854
|
+
async function committedMovement(pool, movementId) {
|
|
855
|
+
var r;
|
|
856
|
+
try {
|
|
857
|
+
r = await query(pool, 'SELECT movement_id, target_table, inserted, replaced, committed_at FROM ' +
|
|
858
|
+
quoteIdent(LOADS_LEDGER) + ' WHERE movement_id = $1 LIMIT 1', [movementId]);
|
|
859
|
+
} catch (err) {
|
|
860
|
+
if (/does not exist/i.test(String(err.message))) return null;
|
|
861
|
+
throw err;
|
|
862
|
+
}
|
|
863
|
+
return r.rows[0] || null;
|
|
864
|
+
}
|
|
865
|
+
|
|
741
866
|
module.exports = {
|
|
742
867
|
requires: ['@duckdb/node-api'],
|
|
743
868
|
|
|
@@ -766,6 +891,13 @@ module.exports = {
|
|
|
766
891
|
|
|
767
892
|
rollbackMovement: rollbackMovement,
|
|
768
893
|
|
|
894
|
+
stagingTableFor: stagingTableFor,
|
|
895
|
+
prepareStaging: prepareStaging,
|
|
896
|
+
dropStaging: dropStaging,
|
|
897
|
+
mergeStaged: mergeStaged,
|
|
898
|
+
committedMovement: committedMovement,
|
|
899
|
+
LOADS_LEDGER: LOADS_LEDGER,
|
|
900
|
+
|
|
769
901
|
INTERNAL_ID_COL: INTERNAL_ID_COL,
|
|
770
902
|
INTERNAL_MOVEMENT_COL: INTERNAL_MOVEMENT_COL,
|
|
771
903
|
|
package/lib/uploader/index.js
CHANGED
|
@@ -28,6 +28,11 @@
|
|
|
28
28
|
// errorTable, // defaults to `<targetTable>_import_errors`
|
|
29
29
|
// batchDir, // where NDJSON files spool to. Defaults to os.tmpdir()/xeplr-actions
|
|
30
30
|
// keepBatchFiles, // default false — clean up NDJSON files after success
|
|
31
|
+
// staged, // default false. true: batches land in <targetTable>__xeplr_staging
|
|
32
|
+
// // and move into the target in ONE transaction at the end, so a
|
|
33
|
+
// // load that stops or dies midway never touched the target.
|
|
34
|
+
// // Needs a driver with mergeStaged (DuckDB). See the driver's
|
|
35
|
+
// // "staged loads" section.
|
|
31
36
|
// onProgress, // optional ({rowsRead, batches}) => void, called per batch.
|
|
32
37
|
// // Rows READ, not rows confirmed written — see onBatch.
|
|
33
38
|
// // Throwing in it cannot fail the movement.
|
|
@@ -76,6 +81,10 @@ async function upload(opts) {
|
|
|
76
81
|
|
|
77
82
|
var spoolResult = null;
|
|
78
83
|
var runDir = null;
|
|
84
|
+
// A staged load's staging table, once made — and whether its rows have moved
|
|
85
|
+
// into the target. Anything else at the end means they never will: dropped.
|
|
86
|
+
var staged = false;
|
|
87
|
+
var merged = null;
|
|
79
88
|
|
|
80
89
|
// STOPPING ON REQUEST (opts.signal, an AbortSignal). The writes still queued
|
|
81
90
|
// are dropped the moment it fires — in-flight ones settle — and the read
|
|
@@ -120,7 +129,11 @@ async function upload(opts) {
|
|
|
120
129
|
var scanLimit = Math.min(sample.length, opts.firstBatchScanRows);
|
|
121
130
|
declared = inferColumns(sample.slice(0, scanLimit));
|
|
122
131
|
}
|
|
123
|
-
columns = await runBootstrap(driver, pool, opts.targetTable, declared, opts.primaryKeys);
|
|
132
|
+
columns = await runBootstrap(driver, pool, opts.targetTable, declared, opts.primaryKeys, opts.staged);
|
|
133
|
+
if (opts.staged) {
|
|
134
|
+
opts.writeTable = await driver.prepareStaging(pool, opts.targetTable);
|
|
135
|
+
staged = true;
|
|
136
|
+
}
|
|
124
137
|
effectiveBatchSize = Math.min(opts.batchSize, safeBatchSize(columns.length, opts.maxParamsPerBatch));
|
|
125
138
|
await opts.metaStore.recordStart(opts.movementId, {
|
|
126
139
|
targetTable: opts.targetTable,
|
|
@@ -221,6 +234,22 @@ async function upload(opts) {
|
|
|
221
234
|
|
|
222
235
|
var stats = opts.queue.stats(opts.movementId) || {};
|
|
223
236
|
var status = stats.aborted ? 'aborted' : 'completed';
|
|
237
|
+
|
|
238
|
+
// THE ONE STEP THAT TOUCHES THE TARGET, for a staged load. Not for an
|
|
239
|
+
// aborted one: its rows are dropped with the staging table, in finally.
|
|
240
|
+
if (staged && !stats.aborted) {
|
|
241
|
+
if (opts.onProgress) {
|
|
242
|
+
try {
|
|
243
|
+
opts.onProgress({ rowsRead: rowsSeen, batches: totalBatches, rowsWritten: writtenSoFar(opts), phase: 'merging' });
|
|
244
|
+
} catch (e) { /* reporting must not fail the movement */ }
|
|
245
|
+
}
|
|
246
|
+
merged = await driver.mergeStaged(pool, {
|
|
247
|
+
targetTable: opts.targetTable,
|
|
248
|
+
columns: columns,
|
|
249
|
+
primaryKeys: opts.primaryKeys,
|
|
250
|
+
movementId: opts.movementId
|
|
251
|
+
});
|
|
252
|
+
}
|
|
224
253
|
await opts.metaStore.recordEnd(opts.movementId, {
|
|
225
254
|
status: status,
|
|
226
255
|
totalRows: summary.totalRows,
|
|
@@ -238,9 +267,16 @@ async function upload(opts) {
|
|
|
238
267
|
completed: stats.completed || 0,
|
|
239
268
|
dropped: stats.dropped || 0,
|
|
240
269
|
aborted: !!stats.aborted,
|
|
270
|
+
// Staged only: what the move put in the target — { inserted, replaced }.
|
|
271
|
+
merged: merged,
|
|
241
272
|
durationMs: Date.now() - startedAt
|
|
242
273
|
};
|
|
243
274
|
} finally {
|
|
275
|
+
// A staged load that did not move its rows: stopped, failed, aborted.
|
|
276
|
+
// Its staging table is all it wrote, and dropping it is the whole undo.
|
|
277
|
+
if (staged && !merged && pool) {
|
|
278
|
+
try { await driver.dropStaging(pool, opts.targetTable); } catch (_) {}
|
|
279
|
+
}
|
|
244
280
|
// Best-effort cleanup: delete the NDJSON spool directory unless caller
|
|
245
281
|
// asked to keep it (e.g. for later resume / audit).
|
|
246
282
|
if (runDir && !opts.keepBatchFiles) {
|
|
@@ -333,6 +369,11 @@ function normalizeOptions(opts) {
|
|
|
333
369
|
opts.batchDir = opts.batchDir ||
|
|
334
370
|
(process.env.XEPLR_ACTIONS_TMP_DIR || path.join(os.tmpdir(), 'xeplr-actions'));
|
|
335
371
|
opts.keepBatchFiles = !!opts.keepBatchFiles;
|
|
372
|
+
opts.staged = !!opts.staged;
|
|
373
|
+
opts.writeTable = null;
|
|
374
|
+
if (opts.staged && typeof opts.driver.mergeStaged !== 'function') {
|
|
375
|
+
throw new Error('upload: staged loads are not supported for this database — its driver has no mergeStaged');
|
|
376
|
+
}
|
|
336
377
|
return opts;
|
|
337
378
|
}
|
|
338
379
|
|
|
@@ -341,7 +382,7 @@ function safeBatchSize(columnCount, maxParams) {
|
|
|
341
382
|
return Math.max(1, Math.floor(maxParams / paramsPerRow));
|
|
342
383
|
}
|
|
343
384
|
|
|
344
|
-
async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
|
|
385
|
+
async function runBootstrap(driver, pool, targetTable, columns, primaryKeys, staged) {
|
|
345
386
|
await driver.query(pool, driver.buildCreateTableSql(targetTable, columns, primaryKeys));
|
|
346
387
|
await driver.query(pool, driver.buildCreateErrorTableSql(targetTable));
|
|
347
388
|
|
|
@@ -349,7 +390,9 @@ async function runBootstrap(driver, pool, targetTable, columns, primaryKeys) {
|
|
|
349
390
|
// per dialect (PG: IF NOT EXISTS; MySQL: introspect-then-DDL; MSSQL:
|
|
350
391
|
// IF OBJECT_ID/COL_LENGTH), so the driver owns it, not this file.
|
|
351
392
|
await driver.ensureMovementColumn(pool, targetTable);
|
|
352
|
-
|
|
393
|
+
// A staged upsert matches keys in its move and needs no index — see the
|
|
394
|
+
// DuckDB driver's "staged loads" section for why it must not have one.
|
|
395
|
+
if (!staged) await driver.ensureUpsertIndex(pool, targetTable, primaryKeys);
|
|
353
396
|
|
|
354
397
|
var targetSchema = await driver.getTableSchema(pool, targetTable);
|
|
355
398
|
var { reconcileColumns } = require('./reconcile');
|
|
@@ -393,9 +436,11 @@ function watchWrites(opts, build, everyMs) {
|
|
|
393
436
|
|
|
394
437
|
async function enqueueBatch(opts, rows, columns, batchRef) {
|
|
395
438
|
var driver = opts.driver;
|
|
396
|
-
|
|
439
|
+
// Staged: plain inserts into the staging table; the keys are matched once,
|
|
440
|
+
// in the move.
|
|
441
|
+
var target = opts.writeTable || opts.targetTable;
|
|
397
442
|
var movementId = opts.movementId;
|
|
398
|
-
var pks = opts.primaryKeys;
|
|
443
|
+
var pks = opts.staged ? null : opts.primaryKeys;
|
|
399
444
|
|
|
400
445
|
function rowsToSql(rs) { return driver.buildInsertSql(target, rs, columns, movementId, pks); }
|
|
401
446
|
|
|
@@ -420,6 +465,26 @@ function looksLikePool(v) {
|
|
|
420
465
|
return v && typeof v.query === 'function' && typeof v.host !== 'string';
|
|
421
466
|
}
|
|
422
467
|
|
|
468
|
+
/**
|
|
469
|
+
* Did a staged movement reach its target? Read from the target's own ledger,
|
|
470
|
+
* written in the same transaction as the move — so for a run that ended
|
|
471
|
+
* without saying (its process was killed) this is the answer, not a guess.
|
|
472
|
+
* Returns the ledger row, or null.
|
|
473
|
+
*/
|
|
474
|
+
async function wasCommitted(opts) {
|
|
475
|
+
if (!opts || !opts.movementId) throw new Error('wasCommitted: movementId is required');
|
|
476
|
+
if (!opts.driver || typeof opts.driver.committedMovement !== 'function') {
|
|
477
|
+
throw new Error('wasCommitted: this database keeps no load ledger');
|
|
478
|
+
}
|
|
479
|
+
var poolIsExternal = looksLikePool(opts.connection);
|
|
480
|
+
var pool = poolIsExternal ? opts.connection : await opts.driver.connect(opts.connection);
|
|
481
|
+
try {
|
|
482
|
+
return await opts.driver.committedMovement(pool, opts.movementId);
|
|
483
|
+
} finally {
|
|
484
|
+
if (!poolIsExternal) await opts.driver.close(pool);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
423
488
|
var { inferColumnType } = require('./infer');
|
|
424
489
|
|
|
425
490
|
module.exports = {
|
|
@@ -427,6 +492,7 @@ module.exports = {
|
|
|
427
492
|
watchWrites: watchWrites,
|
|
428
493
|
stoppedError: stoppedError,
|
|
429
494
|
rollback: rollback,
|
|
495
|
+
wasCommitted: wasCommitted,
|
|
430
496
|
inferColumns: inferColumns,
|
|
431
497
|
inferColumnType: inferColumnType,
|
|
432
498
|
makeMetaStore: require('./meta-store-knex')
|
package/package.json
CHANGED