@zintrust/d1-migrator 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;
|
|
1
|
+
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAysCF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IA+EhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BA0E7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA8BlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAyClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
|
package/dist/cli/DataMigrator.js
CHANGED
|
@@ -147,6 +147,16 @@ const SOURCE_CONNECT_RETRY_ATTEMPTS = 3;
|
|
|
147
147
|
const SOURCE_CONNECT_RETRY_BASE_DELAY_MS = 500;
|
|
148
148
|
const FAILED_TABLE_REPORT_PATH = path.join('logs', 'd1-migration-failed-report.log');
|
|
149
149
|
const REMOTE_SQL_GROW_THRESHOLD_RATIO = 0.6;
|
|
150
|
+
const getEnvBatchSize = () => {
|
|
151
|
+
const envVal = process.env['MIGRATE_TO_D1_BATCH_SIZE'] || process.env['D1_MIGRATOR_BATCH_SIZE'];
|
|
152
|
+
if (envVal) {
|
|
153
|
+
const parsed = Number.parseInt(envVal, 10);
|
|
154
|
+
if (!Number.isNaN(parsed) && parsed > 0) {
|
|
155
|
+
return parsed;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return undefined;
|
|
159
|
+
};
|
|
150
160
|
const formatDuration = (durationMs) => {
|
|
151
161
|
if (durationMs < 1000) {
|
|
152
162
|
return `${durationMs}ms`;
|
|
@@ -180,14 +190,16 @@ const normalizeSourceBatchSize = (requestedBatchSize) => {
|
|
|
180
190
|
return Math.max(MIN_SOURCE_READ_BATCH_SIZE, requestedBatchSize);
|
|
181
191
|
};
|
|
182
192
|
const createSourceBatchTuning = (initialBatchSize) => {
|
|
193
|
+
const finalSize = getEnvBatchSize() ?? initialBatchSize;
|
|
183
194
|
return {
|
|
184
|
-
readBatchSize: normalizeSourceBatchSize(
|
|
195
|
+
readBatchSize: normalizeSourceBatchSize(finalSize),
|
|
185
196
|
readBatchHistory: [],
|
|
186
197
|
};
|
|
187
198
|
};
|
|
188
|
-
const createRemoteBatchTuning = () => {
|
|
199
|
+
const createRemoteBatchTuning = (initialBatchSize) => {
|
|
200
|
+
const finalSize = initialBatchSize ?? getEnvBatchSize() ?? REMOTE_INSERT_ROWS_PER_STATEMENT;
|
|
189
201
|
return {
|
|
190
|
-
rowsPerStatement:
|
|
202
|
+
rowsPerStatement: Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, finalSize),
|
|
191
203
|
maxStatementSqlLength: MAX_REMOTE_INSERT_SQL_LENGTH,
|
|
192
204
|
maxExecutionSqlLength: MAX_REMOTE_EXECUTION_SQL_LENGTH,
|
|
193
205
|
};
|
|
@@ -277,18 +289,18 @@ const reduceSourceBatchSizeAfterTimeout = (connection, batchSize) => {
|
|
|
277
289
|
Logger.warn(`[DataMigrator] Source batch retry: ${currentSize} -> ${reducedSize} rows after retryable read failure`);
|
|
278
290
|
return reducedSize;
|
|
279
291
|
};
|
|
280
|
-
const getRemoteBatchTuning = (connection) => {
|
|
292
|
+
const getRemoteBatchTuning = (connection, initialBatchSize) => {
|
|
281
293
|
if (connection.remoteBatchTuning !== undefined) {
|
|
282
294
|
return connection.remoteBatchTuning;
|
|
283
295
|
}
|
|
284
|
-
connection.remoteBatchTuning = createRemoteBatchTuning();
|
|
296
|
+
connection.remoteBatchTuning = createRemoteBatchTuning(initialBatchSize);
|
|
285
297
|
return connection.remoteBatchTuning;
|
|
286
298
|
};
|
|
287
|
-
const resetRemoteBatchTuning = (connection) => {
|
|
299
|
+
const resetRemoteBatchTuning = (connection, initialBatchSize) => {
|
|
288
300
|
if (connection.type !== 'd1-remote') {
|
|
289
301
|
return;
|
|
290
302
|
}
|
|
291
|
-
connection.remoteBatchTuning = createRemoteBatchTuning();
|
|
303
|
+
connection.remoteBatchTuning = createRemoteBatchTuning(initialBatchSize);
|
|
292
304
|
};
|
|
293
305
|
const getInsertBatchSettings = (connection) => {
|
|
294
306
|
if (connection.type === 'd1-remote') {
|
|
@@ -1066,7 +1078,7 @@ export const DataMigrator = Object.freeze({
|
|
|
1066
1078
|
const totalRows = table.rowCount || 0;
|
|
1067
1079
|
const batchSize = config.batchSize || 1000;
|
|
1068
1080
|
resetSourceBatchTuning(sourceConnection, batchSize);
|
|
1069
|
-
resetRemoteBatchTuning(targetConnection);
|
|
1081
|
+
resetRemoteBatchTuning(targetConnection, batchSize);
|
|
1070
1082
|
const targetRowCount = await DataMigrator.getTargetRowCount(targetConnection, table.name);
|
|
1071
1083
|
if (targetRowCount >= totalRows) {
|
|
1072
1084
|
Logger.info(`Table ${table.name} already synced: ${targetRowCount}/${totalRows} rows, skipping`);
|