@zintrust/d1-migrator 1.9.9 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/DataMigrator.d.ts.map +1 -1
- package/dist/cli/DataMigrator.js +181 -123
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAqrCF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IA+EhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BA0E7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA8BlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAyClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
|
package/dist/cli/DataMigrator.js
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
* Handles the actual data migration between databases
|
|
5
5
|
*/
|
|
6
6
|
import { ErrorFactory, LocalD1Resolver, Logger, WranglerD1 } from '@zintrust/core';
|
|
7
|
+
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
8
|
+
import * as path from 'node:path';
|
|
7
9
|
import { MySQLAdapter } from '@zintrust/db-mysql';
|
|
8
10
|
import { PostgreSQLAdapter } from '@zintrust/db-postgres';
|
|
9
11
|
import { SQLiteAdapter } from '@zintrust/db-sqlite';
|
|
@@ -134,17 +136,17 @@ const bindSqlParameters = (sql, parameters) => {
|
|
|
134
136
|
};
|
|
135
137
|
const REMOTE_INSERT_ROWS_PER_STATEMENT = 1000;
|
|
136
138
|
const LOCAL_INSERT_ROWS_PER_STATEMENT = 500;
|
|
137
|
-
const
|
|
138
|
-
const
|
|
139
|
-
const
|
|
140
|
-
const
|
|
141
|
-
const DEFAULT_REMOTE_TABLE_PARALLELISM =
|
|
142
|
-
const DEFAULT_SOURCE_READ_BATCH_SIZE =
|
|
143
|
-
const MIN_SOURCE_READ_BATCH_SIZE =
|
|
144
|
-
const MAX_SOURCE_READ_BATCH_SIZE = 1500;
|
|
145
|
-
const SOURCE_READ_TARGET_TIME_MS = 5000;
|
|
139
|
+
const MAX_REMOTE_INSERT_ROWS_PER_STATEMENT = 2000;
|
|
140
|
+
const MAX_REMOTE_INSERT_SQL_LENGTH = 95 * 1024;
|
|
141
|
+
const MAX_REMOTE_EXECUTION_SQL_LENGTH = MAX_REMOTE_INSERT_SQL_LENGTH;
|
|
142
|
+
const MIN_REMOTE_INSERT_ROWS_PER_STATEMENT = 1;
|
|
143
|
+
const DEFAULT_REMOTE_TABLE_PARALLELISM = 1;
|
|
144
|
+
const DEFAULT_SOURCE_READ_BATCH_SIZE = 1000;
|
|
145
|
+
const MIN_SOURCE_READ_BATCH_SIZE = 1;
|
|
146
146
|
const SOURCE_CONNECT_RETRY_ATTEMPTS = 3;
|
|
147
147
|
const SOURCE_CONNECT_RETRY_BASE_DELAY_MS = 500;
|
|
148
|
+
const FAILED_TABLE_REPORT_PATH = path.join('logs', 'd1-migration-failed-report.log');
|
|
149
|
+
const REMOTE_SQL_GROW_THRESHOLD_RATIO = 0.6;
|
|
148
150
|
const formatDuration = (durationMs) => {
|
|
149
151
|
if (durationMs < 1000) {
|
|
150
152
|
return `${durationMs}ms`;
|
|
@@ -158,6 +160,9 @@ const formatRowsPerSecond = (rows, durationMs) => {
|
|
|
158
160
|
const rate = rows / (durationMs / 1000);
|
|
159
161
|
return `${rate >= 100 ? rate.toFixed(0) : rate.toFixed(2)} rows/s`;
|
|
160
162
|
};
|
|
163
|
+
const getSqlByteLength = (value) => {
|
|
164
|
+
return Buffer.byteLength(value, 'utf8');
|
|
165
|
+
};
|
|
161
166
|
const waitMs = async (durationMs) => {
|
|
162
167
|
await new Promise((resolve) => {
|
|
163
168
|
const start = Date.now();
|
|
@@ -171,24 +176,55 @@ const waitMs = async (durationMs) => {
|
|
|
171
176
|
poll();
|
|
172
177
|
});
|
|
173
178
|
};
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
179
|
+
const normalizeSourceBatchSize = (requestedBatchSize) => {
|
|
180
|
+
return Math.max(MIN_SOURCE_READ_BATCH_SIZE, requestedBatchSize);
|
|
181
|
+
};
|
|
182
|
+
const createSourceBatchTuning = (initialBatchSize) => {
|
|
183
|
+
return {
|
|
184
|
+
readBatchSize: normalizeSourceBatchSize(initialBatchSize),
|
|
185
|
+
readBatchHistory: [],
|
|
186
|
+
};
|
|
187
|
+
};
|
|
188
|
+
const createRemoteBatchTuning = () => {
|
|
189
|
+
return {
|
|
190
|
+
rowsPerStatement: REMOTE_INSERT_ROWS_PER_STATEMENT,
|
|
191
|
+
maxStatementSqlLength: MAX_REMOTE_INSERT_SQL_LENGTH,
|
|
192
|
+
maxExecutionSqlLength: MAX_REMOTE_EXECUTION_SQL_LENGTH,
|
|
193
|
+
};
|
|
194
|
+
};
|
|
195
|
+
const sanitizeFailureReason = (reason) => {
|
|
196
|
+
return reason.replaceAll(/\s+/g, ' ').trim();
|
|
197
|
+
};
|
|
198
|
+
const appendFailedTableReport = (params) => {
|
|
199
|
+
const reportPath = path.join(process.cwd(), FAILED_TABLE_REPORT_PATH);
|
|
200
|
+
const reportDir = path.dirname(reportPath);
|
|
201
|
+
const reason = sanitizeFailureReason(params.errors.at(-1) ?? 'Unknown migration failure');
|
|
202
|
+
const entry = JSON.stringify({
|
|
203
|
+
timestamp: new Date().toISOString(),
|
|
204
|
+
migrationId: params.migrationId,
|
|
205
|
+
targetDatabase: params.targetDatabase,
|
|
206
|
+
tableName: params.tableName,
|
|
207
|
+
rowsMigrated: params.rowsMigrated,
|
|
208
|
+
totalRows: params.totalRows,
|
|
209
|
+
reason,
|
|
210
|
+
});
|
|
211
|
+
mkdirSync(reportDir, { recursive: true });
|
|
212
|
+
appendFileSync(reportPath, `${entry}\n`, 'utf8');
|
|
213
|
+
Logger.warn(`[DataMigrator] Skipped remaining rows for ${params.tableName}. Failure recorded in ${FAILED_TABLE_REPORT_PATH}`);
|
|
180
214
|
};
|
|
181
215
|
const getSourceBatchTuning = (connection, initialBatchSize) => {
|
|
182
216
|
if (connection.sourceBatchTuning !== undefined) {
|
|
183
217
|
return connection.sourceBatchTuning;
|
|
184
218
|
}
|
|
185
|
-
|
|
186
|
-
connection.sourceBatchTuning = {
|
|
187
|
-
readBatchSize: startingBatchSize,
|
|
188
|
-
readBatchHistory: [],
|
|
189
|
-
};
|
|
219
|
+
connection.sourceBatchTuning = createSourceBatchTuning(initialBatchSize);
|
|
190
220
|
return connection.sourceBatchTuning;
|
|
191
221
|
};
|
|
222
|
+
const resetSourceBatchTuning = (connection, initialBatchSize) => {
|
|
223
|
+
if (connection.driver !== 'mysql') {
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
connection.sourceBatchTuning = createSourceBatchTuning(initialBatchSize);
|
|
227
|
+
};
|
|
192
228
|
const isRetryableConnectionError = (error) => {
|
|
193
229
|
const message = getErrorMessage(error).toLowerCase();
|
|
194
230
|
return (message.includes('etimedout') ||
|
|
@@ -229,35 +265,31 @@ const updateSourceBatchTuningAfterSuccess = (connection, rowsRead, durationMs, b
|
|
|
229
265
|
if (tuning.readBatchHistory.length > 10) {
|
|
230
266
|
tuning.readBatchHistory.shift();
|
|
231
267
|
}
|
|
232
|
-
const avgTimePerRow = tuning.readBatchHistory.reduce((sum, entry) => sum + entry.timePerRow, 0) /
|
|
233
|
-
tuning.readBatchHistory.length;
|
|
234
|
-
const targetTimePerBatchMs = Math.max(1000, Math.min(SOURCE_READ_TARGET_TIME_MS, Math.floor(batchSize * 5)));
|
|
235
|
-
const optimalSize = calculateOptimalBatchSize(avgTimePerRow, targetTimePerBatchMs, MIN_SOURCE_READ_BATCH_SIZE, MAX_SOURCE_READ_BATCH_SIZE);
|
|
236
|
-
if (Math.abs(optimalSize - tuning.readBatchSize) >= 50) {
|
|
237
|
-
tuning.readBatchSize = optimalSize;
|
|
238
|
-
Logger.info(`[DataMigrator] Adaptive source batching: adjusted to ${optimalSize} rows (avg ${avgTimePerRow.toFixed(2)}ms/row)`);
|
|
239
|
-
}
|
|
240
268
|
};
|
|
241
269
|
const reduceSourceBatchSizeAfterTimeout = (connection, batchSize) => {
|
|
242
270
|
const tuning = getSourceBatchTuning(connection, batchSize);
|
|
243
|
-
const
|
|
244
|
-
if (
|
|
245
|
-
|
|
246
|
-
Logger.warn(`[DataMigrator] Adaptive source batching: reduced to ${reducedSize} rows after timeout-like failure`);
|
|
271
|
+
const currentSize = Math.min(tuning.readBatchSize, batchSize);
|
|
272
|
+
if (currentSize <= MIN_SOURCE_READ_BATCH_SIZE) {
|
|
273
|
+
return 0;
|
|
247
274
|
}
|
|
275
|
+
const reducedSize = Math.max(MIN_SOURCE_READ_BATCH_SIZE, Math.floor(currentSize / 2));
|
|
276
|
+
tuning.readBatchSize = reducedSize;
|
|
277
|
+
Logger.warn(`[DataMigrator] Source batch retry: ${currentSize} -> ${reducedSize} rows after retryable read failure`);
|
|
248
278
|
return reducedSize;
|
|
249
279
|
};
|
|
250
280
|
const getRemoteBatchTuning = (connection) => {
|
|
251
281
|
if (connection.remoteBatchTuning !== undefined) {
|
|
252
282
|
return connection.remoteBatchTuning;
|
|
253
283
|
}
|
|
254
|
-
connection.remoteBatchTuning =
|
|
255
|
-
rowsPerStatement: REMOTE_INSERT_ROWS_PER_STATEMENT,
|
|
256
|
-
maxStatementSqlLength: MAX_REMOTE_INSERT_SQL_LENGTH,
|
|
257
|
-
maxExecutionSqlLength: MAX_REMOTE_EXECUTION_SQL_LENGTH,
|
|
258
|
-
};
|
|
284
|
+
connection.remoteBatchTuning = createRemoteBatchTuning();
|
|
259
285
|
return connection.remoteBatchTuning;
|
|
260
286
|
};
|
|
287
|
+
const resetRemoteBatchTuning = (connection) => {
|
|
288
|
+
if (connection.type !== 'd1-remote') {
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
connection.remoteBatchTuning = createRemoteBatchTuning();
|
|
292
|
+
};
|
|
261
293
|
const getInsertBatchSettings = (connection) => {
|
|
262
294
|
if (connection.type === 'd1-remote') {
|
|
263
295
|
const tuning = getRemoteBatchTuning(connection);
|
|
@@ -279,33 +311,32 @@ const adjustRemoteBatchTuning = (connection, executedRows, durationMs, sqlLength
|
|
|
279
311
|
}
|
|
280
312
|
const tuning = getRemoteBatchTuning(connection);
|
|
281
313
|
const previousRowsPerStatement = tuning.rowsPerStatement;
|
|
282
|
-
const
|
|
283
|
-
if (
|
|
284
|
-
executedRows
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
else if (durationMs >= 10000 || nearSqlLimit) {
|
|
289
|
-
tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement - 50, Math.floor(previousRowsPerStatement * 0.8)));
|
|
314
|
+
const growthThreshold = Math.floor(tuning.maxStatementSqlLength * REMOTE_SQL_GROW_THRESHOLD_RATIO);
|
|
315
|
+
if (previousRowsPerStatement >= MAX_REMOTE_INSERT_ROWS_PER_STATEMENT ||
|
|
316
|
+
executedRows < previousRowsPerStatement ||
|
|
317
|
+
sqlLength > growthThreshold ||
|
|
318
|
+
durationMs > 10000) {
|
|
319
|
+
return;
|
|
290
320
|
}
|
|
321
|
+
tuning.rowsPerStatement = Math.min(MAX_REMOTE_INSERT_ROWS_PER_STATEMENT, previousRowsPerStatement * 2);
|
|
291
322
|
if (tuning.rowsPerStatement !== previousRowsPerStatement) {
|
|
292
|
-
Logger.info(`[DataMigrator]
|
|
323
|
+
Logger.info(`[DataMigrator] Remote batch growth: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after ${executedRows} rows in ${formatDuration(durationMs)} with sql_size=${sqlLength}/${tuning.maxStatementSqlLength} bytes`);
|
|
293
324
|
}
|
|
294
325
|
};
|
|
295
|
-
const reduceRemoteBatchTuningAfterFailure = (connection
|
|
326
|
+
const reduceRemoteBatchTuningAfterFailure = (connection) => {
|
|
296
327
|
if (connection.type !== 'd1-remote') {
|
|
297
|
-
return;
|
|
328
|
+
return 0;
|
|
298
329
|
}
|
|
299
330
|
const tuning = getRemoteBatchTuning(connection);
|
|
300
331
|
const previousRowsPerStatement = tuning.rowsPerStatement;
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
return;
|
|
332
|
+
if (previousRowsPerStatement <= MIN_REMOTE_INSERT_ROWS_PER_STATEMENT) {
|
|
333
|
+
return 0;
|
|
304
334
|
}
|
|
305
|
-
tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.
|
|
335
|
+
tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.floor(previousRowsPerStatement / 2));
|
|
306
336
|
if (tuning.rowsPerStatement !== previousRowsPerStatement) {
|
|
307
337
|
Logger.warn(`[DataMigrator] Adaptive remote batching: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after failed remote insert`);
|
|
308
338
|
}
|
|
339
|
+
return tuning.rowsPerStatement;
|
|
309
340
|
};
|
|
310
341
|
const logTableMigrationProgress = (table, rowsMigrated, totalRows, batchSize) => {
|
|
311
342
|
if (totalRows <= 10000 || rowsMigrated % (batchSize * 10) !== 0) {
|
|
@@ -407,18 +438,20 @@ const buildChunkProcessingFailureResult = (sourceConnection, offset, batchSize,
|
|
|
407
438
|
errors.push(errorMsg);
|
|
408
439
|
if (sourceConnection.driver === 'mysql' && isRetryableConnectionError(error)) {
|
|
409
440
|
const reducedBatchSize = reduceSourceBatchSizeAfterTimeout(sourceConnection, batchSize);
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
441
|
+
if (reducedBatchSize > 0) {
|
|
442
|
+
return {
|
|
443
|
+
rowsMigrated: 0,
|
|
444
|
+
nextOffset: offset,
|
|
445
|
+
nextBatchSize: reducedBatchSize,
|
|
446
|
+
continueProcessing: true,
|
|
447
|
+
};
|
|
448
|
+
}
|
|
416
449
|
}
|
|
417
450
|
return {
|
|
418
451
|
rowsMigrated: 0,
|
|
419
|
-
nextOffset: offset
|
|
452
|
+
nextOffset: offset,
|
|
420
453
|
nextBatchSize: batchSize,
|
|
421
|
-
continueProcessing:
|
|
454
|
+
continueProcessing: false,
|
|
422
455
|
};
|
|
423
456
|
};
|
|
424
457
|
const processTableChunk = async (table, sourceConnection, targetConnection, offset, batchSize, errors) => {
|
|
@@ -503,7 +536,7 @@ const executeWithConcurrency = async (items, concurrency, worker) => {
|
|
|
503
536
|
const estimateRemoteRowSqlLength = (keys, row) => {
|
|
504
537
|
const delimitersLength = keys.length > 0 ? (keys.length - 1) * 2 : 0;
|
|
505
538
|
const valuesLength = keys.reduce((total, key) => {
|
|
506
|
-
return total + toSqlLiteral(row[key])
|
|
539
|
+
return total + getSqlByteLength(toSqlLiteral(row[key]));
|
|
507
540
|
}, 0);
|
|
508
541
|
return valuesLength + delimitersLength + 2;
|
|
509
542
|
};
|
|
@@ -520,7 +553,7 @@ const createInsertStatements = (targetType, settings, tableName, data) => {
|
|
|
520
553
|
const statements = [];
|
|
521
554
|
let batchRows = [];
|
|
522
555
|
let batchParameters = [];
|
|
523
|
-
let batchSqlLength = prefix
|
|
556
|
+
let batchSqlLength = getSqlByteLength(prefix);
|
|
524
557
|
const flushBatch = () => {
|
|
525
558
|
if (batchRows.length === 0) {
|
|
526
559
|
return;
|
|
@@ -529,14 +562,17 @@ const createInsertStatements = (targetType, settings, tableName, data) => {
|
|
|
529
562
|
sql: `${prefix}${batchRows.map(() => rowPlaceholder).join(', ')}`,
|
|
530
563
|
parameters: batchParameters,
|
|
531
564
|
rowCount: batchRows.length,
|
|
565
|
+
rows: batchRows,
|
|
532
566
|
});
|
|
533
567
|
batchRows = [];
|
|
534
568
|
batchParameters = [];
|
|
535
|
-
batchSqlLength = prefix
|
|
569
|
+
batchSqlLength = getSqlByteLength(prefix);
|
|
536
570
|
};
|
|
537
571
|
for (const row of data) {
|
|
538
572
|
const rowParameters = keys.map((key) => row[key]);
|
|
539
|
-
const rowSqlLength = targetType === 'd1-remote'
|
|
573
|
+
const rowSqlLength = targetType === 'd1-remote'
|
|
574
|
+
? estimateRemoteRowSqlLength(keys, row)
|
|
575
|
+
: getSqlByteLength(rowPlaceholder);
|
|
540
576
|
const separatorLength = batchRows.length > 0 ? 2 : 0;
|
|
541
577
|
const nextSqlLength = batchSqlLength + separatorLength + rowSqlLength;
|
|
542
578
|
if (batchRows.length > 0 && (batchRows.length >= rowLimit || nextSqlLength > maxSqlLength)) {
|
|
@@ -549,42 +585,58 @@ const createInsertStatements = (targetType, settings, tableName, data) => {
|
|
|
549
585
|
flushBatch();
|
|
550
586
|
return statements;
|
|
551
587
|
};
|
|
552
|
-
const
|
|
553
|
-
if (
|
|
554
|
-
|
|
555
|
-
}
|
|
556
|
-
const
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
588
|
+
const executeInsertStatement = async (targetConnection, statement) => {
|
|
589
|
+
if (!targetConnection.adapter) {
|
|
590
|
+
throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
|
|
591
|
+
}
|
|
592
|
+
const executionStartTime = Date.now();
|
|
593
|
+
const result = await targetConnection.adapter.query(statement.sql, statement.parameters);
|
|
594
|
+
const executionDurationMs = Date.now() - executionStartTime;
|
|
595
|
+
const affectedRows = typeof result.rowCount === 'number' ? result.rowCount : statement.rowCount;
|
|
596
|
+
return { affectedRows, executionDurationMs };
|
|
597
|
+
};
|
|
598
|
+
const createRemoteInsertExecutionQueue = (targetConnection, tableName, rows) => {
|
|
599
|
+
const batchSettings = getInsertBatchSettings(targetConnection);
|
|
600
|
+
return createInsertStatements(targetConnection.type, batchSettings, tableName, rows);
|
|
601
|
+
};
|
|
602
|
+
const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
|
|
603
|
+
const pendingRows = [data];
|
|
604
|
+
let insertedRows = 0;
|
|
605
|
+
while (pendingRows.length > 0) {
|
|
606
|
+
const currentRows = pendingRows.shift();
|
|
607
|
+
if (currentRows === undefined || currentRows.length === 0) {
|
|
608
|
+
continue;
|
|
564
609
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
610
|
+
const statements = createRemoteInsertExecutionQueue(targetConnection, tableName, currentRows);
|
|
611
|
+
let shouldRetryCurrentRows = false;
|
|
612
|
+
for (const [statementIndex, statement] of statements.entries()) {
|
|
613
|
+
try {
|
|
614
|
+
const { affectedRows, executionDurationMs } = await executeInsertStatement(targetConnection, statement);
|
|
615
|
+
insertedRows += affectedRows;
|
|
616
|
+
adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, getSqlByteLength(statement.sql));
|
|
617
|
+
}
|
|
618
|
+
catch (error) {
|
|
619
|
+
const remainingGroups = statements
|
|
620
|
+
.slice(statementIndex + 1)
|
|
621
|
+
.map((remainingStatement) => remainingStatement.rows)
|
|
622
|
+
.filter((remainingRows) => remainingRows.length > 0);
|
|
623
|
+
const nextRowsPerStatement = reduceRemoteBatchTuningAfterFailure(targetConnection);
|
|
624
|
+
if (nextRowsPerStatement <= 0 || statement.rows.length <= 1) {
|
|
625
|
+
throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
|
|
626
|
+
rowCount: statement.rowCount,
|
|
627
|
+
reason: getErrorMessage(error),
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
pendingRows.unshift(statement.rows, ...remainingGroups);
|
|
631
|
+
shouldRetryCurrentRows = true;
|
|
632
|
+
break;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
if (shouldRetryCurrentRows) {
|
|
636
|
+
continue;
|
|
580
637
|
}
|
|
581
|
-
sqlParts.push(statement.sql);
|
|
582
|
-
parameters.push(...statement.parameters);
|
|
583
|
-
rowCount += statement.rowCount;
|
|
584
|
-
currentLength += (sqlParts.length > 1 ? 2 : 0) + statement.sql.length;
|
|
585
638
|
}
|
|
586
|
-
|
|
587
|
-
return batches;
|
|
639
|
+
return insertedRows;
|
|
588
640
|
};
|
|
589
641
|
const createRemoteD1Adapter = (database) => {
|
|
590
642
|
return {
|
|
@@ -938,7 +990,10 @@ export const DataMigrator = Object.freeze({
|
|
|
938
990
|
schemaStatements.length = 0;
|
|
939
991
|
};
|
|
940
992
|
const pushSchemaStatement = async (sql) => {
|
|
941
|
-
const trimmedSql = sql.trim();
|
|
993
|
+
const trimmedSql = sql.trim().replace(/;+$/u, '');
|
|
994
|
+
if (trimmedSql === '') {
|
|
995
|
+
return;
|
|
996
|
+
}
|
|
942
997
|
const nextLength = schemaStatements.length === 0
|
|
943
998
|
? trimmedSql.length
|
|
944
999
|
: schemaStatements.join(';\n').length + 2 + trimmedSql.length;
|
|
@@ -1009,6 +1064,8 @@ export const DataMigrator = Object.freeze({
|
|
|
1009
1064
|
try {
|
|
1010
1065
|
const totalRows = table.rowCount || 0;
|
|
1011
1066
|
const batchSize = config.batchSize || 1000;
|
|
1067
|
+
resetSourceBatchTuning(sourceConnection, batchSize);
|
|
1068
|
+
resetRemoteBatchTuning(targetConnection);
|
|
1012
1069
|
const targetRowCount = await DataMigrator.getTargetRowCount(targetConnection, table.name);
|
|
1013
1070
|
if (targetRowCount >= totalRows) {
|
|
1014
1071
|
Logger.info(`Table ${table.name} already synced: ${targetRowCount}/${totalRows} rows, skipping`);
|
|
@@ -1021,6 +1078,16 @@ export const DataMigrator = Object.freeze({
|
|
|
1021
1078
|
Logger.info(`Processing ${totalRows} rows in batches of ${batchSize}`);
|
|
1022
1079
|
}
|
|
1023
1080
|
rowsMigrated = await DataMigrator.processTableChunks(table, sourceConnection, targetConnection, totalRows, batchSize, targetRowCount, errors);
|
|
1081
|
+
if (errors.length > 0 && rowsMigrated < totalRows) {
|
|
1082
|
+
appendFailedTableReport({
|
|
1083
|
+
migrationId: config.migrationId || 'unknown',
|
|
1084
|
+
targetDatabase: targetConnection.database,
|
|
1085
|
+
tableName: table.name,
|
|
1086
|
+
rowsMigrated,
|
|
1087
|
+
totalRows,
|
|
1088
|
+
errors,
|
|
1089
|
+
});
|
|
1090
|
+
}
|
|
1024
1091
|
const tableDurationMs = Date.now() - tableStartTime;
|
|
1025
1092
|
Logger.info(`[DataMigrator] Table ${table.name} completed rows=${rowsMigrated}/${totalRows} duration=${formatDuration(tableDurationMs)} rate=${formatRowsPerSecond(rowsMigrated, tableDurationMs)}`);
|
|
1026
1093
|
return { rowsMigrated, errors };
|
|
@@ -1029,6 +1096,14 @@ export const DataMigrator = Object.freeze({
|
|
|
1029
1096
|
const errorMsg = `Table migration failed for ${table.name}: ${error}`;
|
|
1030
1097
|
Logger.error(errorMsg);
|
|
1031
1098
|
errors.push(errorMsg);
|
|
1099
|
+
appendFailedTableReport({
|
|
1100
|
+
migrationId: config.migrationId || 'unknown',
|
|
1101
|
+
targetDatabase: targetConnection.database,
|
|
1102
|
+
tableName: table.name,
|
|
1103
|
+
rowsMigrated,
|
|
1104
|
+
totalRows: table.rowCount || 0,
|
|
1105
|
+
errors,
|
|
1106
|
+
});
|
|
1032
1107
|
return { rowsMigrated, errors };
|
|
1033
1108
|
}
|
|
1034
1109
|
},
|
|
@@ -1043,8 +1118,7 @@ export const DataMigrator = Object.freeze({
|
|
|
1043
1118
|
state.currentBatchSize = getTableChunkBatchSize(sourceConnection, state.retryBatchSize);
|
|
1044
1119
|
const result = await processTableChunk(table, sourceConnection, targetConnection, state.offset, state.currentBatchSize, errors);
|
|
1045
1120
|
state = applyTableChunkResult(state, result, batchSize, table, totalRows);
|
|
1046
|
-
if (!result.continueProcessing
|
|
1047
|
-
!(result.rowsMigrated === 0 && result.nextBatchSize < batchSize)) {
|
|
1121
|
+
if (!result.continueProcessing) {
|
|
1048
1122
|
break;
|
|
1049
1123
|
}
|
|
1050
1124
|
}
|
|
@@ -1118,33 +1192,17 @@ export const DataMigrator = Object.freeze({
|
|
|
1118
1192
|
if (!targetConnection.adapter) {
|
|
1119
1193
|
throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
|
|
1120
1194
|
}
|
|
1195
|
+
if (targetConnection.type === 'd1-remote') {
|
|
1196
|
+
return insertRemoteRowsWithRetry(targetConnection, tableName, data);
|
|
1197
|
+
}
|
|
1121
1198
|
const batchSettings = getInsertBatchSettings(targetConnection);
|
|
1122
1199
|
const statements = createInsertStatements(targetConnection.type, batchSettings, tableName, data);
|
|
1123
|
-
const executableStatements =
|
|
1124
|
-
? createRemoteExecutionBatchesWithLimit(statements, batchSettings.maxExecutionSqlLength)
|
|
1125
|
-
: statements;
|
|
1200
|
+
const executableStatements = statements;
|
|
1126
1201
|
let insertedRows = 0;
|
|
1127
1202
|
for (const statement of executableStatements) {
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
const executionDurationMs = Date.now() - executionStartTime;
|
|
1132
|
-
const affectedRows = typeof result.rowCount === 'number' ? result.rowCount : statement.rowCount;
|
|
1133
|
-
insertedRows += affectedRows;
|
|
1134
|
-
adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, statement.sql.length);
|
|
1135
|
-
}
|
|
1136
|
-
catch {
|
|
1137
|
-
if (targetConnection.type === 'd1-remote' && data.length > 1 && statement.rowCount > 1) {
|
|
1138
|
-
reduceRemoteBatchTuningAfterFailure(targetConnection, statement.sql.length);
|
|
1139
|
-
const midpoint = Math.max(1, Math.floor(data.length / 2));
|
|
1140
|
-
const left = await DataMigrator.insertData(targetConnection, tableName, data.slice(0, midpoint));
|
|
1141
|
-
const right = await DataMigrator.insertData(targetConnection, tableName, data.slice(midpoint));
|
|
1142
|
-
return left + right;
|
|
1143
|
-
}
|
|
1144
|
-
throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
|
|
1145
|
-
rowCount: statement.rowCount,
|
|
1146
|
-
});
|
|
1147
|
-
}
|
|
1203
|
+
const { affectedRows, executionDurationMs } = await executeInsertStatement(targetConnection, statement);
|
|
1204
|
+
insertedRows += affectedRows;
|
|
1205
|
+
adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, getSqlByteLength(statement.sql));
|
|
1148
1206
|
}
|
|
1149
1207
|
return insertedRows;
|
|
1150
1208
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/d1-migrator",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.1",
|
|
4
4
|
"description": "Resumable database migration toolkit for moving data to Cloudflare D1 with ZinTrust.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"type": "module",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"@zintrust/db-d1": "^1.8.0",
|
|
44
|
-
"@zintrust/db-mysql": "^2.0.
|
|
44
|
+
"@zintrust/db-mysql": "^2.0.1",
|
|
45
45
|
"@zintrust/db-postgres": "^1.8.0",
|
|
46
46
|
"@zintrust/db-sqlite": "^1.8.0",
|
|
47
47
|
"@zintrust/db-sqlserver": "^1.8.0"
|