@zintrust/d1-migrator 1.9.3 → 1.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/DataMigrator.d.ts +10 -0
- package/dist/cli/DataMigrator.d.ts.map +1 -1
- package/dist/cli/DataMigrator.js +320 -55
- package/dist/cli/MigrateToD1Command.d.ts.map +1 -1
- package/dist/cli/MigrateToD1Command.js +10 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -13,7 +13,17 @@ export interface SourceConnection {
|
|
|
13
13
|
sourceSsl?: boolean;
|
|
14
14
|
connected: boolean;
|
|
15
15
|
adapter?: DatabaseAdapter;
|
|
16
|
+
sourceBatchTuning?: SourceBatchTuning;
|
|
16
17
|
}
|
|
18
|
+
type SourceBatchHistoryEntry = {
|
|
19
|
+
rowsRead: number;
|
|
20
|
+
durationMs: number;
|
|
21
|
+
timePerRow: number;
|
|
22
|
+
};
|
|
23
|
+
type SourceBatchTuning = {
|
|
24
|
+
readBatchSize: number;
|
|
25
|
+
readBatchHistory: SourceBatchHistoryEntry[];
|
|
26
|
+
};
|
|
17
27
|
export interface TargetConnection {
|
|
18
28
|
type: 'd1' | 'd1-remote';
|
|
19
29
|
database: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAQD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAqnCF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IA4EhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BAoD7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IAiClB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IA+DlB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
|
package/dist/cli/DataMigrator.js
CHANGED
|
@@ -97,10 +97,10 @@ const toSqlLiteral = (value) => {
|
|
|
97
97
|
return 'NULL';
|
|
98
98
|
}
|
|
99
99
|
if (value instanceof Date) {
|
|
100
|
-
return `'${value.toISOString().
|
|
100
|
+
return `'${value.toISOString().replaceAll("'", "''")}'`;
|
|
101
101
|
}
|
|
102
102
|
if (typeof value === 'string') {
|
|
103
|
-
return `'${value.
|
|
103
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
104
104
|
}
|
|
105
105
|
if (typeof value === 'number') {
|
|
106
106
|
if (!Number.isFinite(value)) {
|
|
@@ -119,11 +119,11 @@ const toSqlLiteral = (value) => {
|
|
|
119
119
|
const bytes = value instanceof Uint8Array ? value : new Uint8Array(value);
|
|
120
120
|
return `X'${toHex(bytes)}'`;
|
|
121
121
|
}
|
|
122
|
-
return `'${JSON.stringify(value).
|
|
122
|
+
return `'${JSON.stringify(value).replaceAll("'", "''")}'`;
|
|
123
123
|
};
|
|
124
124
|
const bindSqlParameters = (sql, parameters) => {
|
|
125
125
|
let index = 0;
|
|
126
|
-
return sql.
|
|
126
|
+
return sql.replaceAll('?', () => {
|
|
127
127
|
if (index >= parameters.length) {
|
|
128
128
|
throw ErrorFactory.createValidationError('Remote D1 SQL parameter count mismatch');
|
|
129
129
|
}
|
|
@@ -132,13 +132,19 @@ const bindSqlParameters = (sql, parameters) => {
|
|
|
132
132
|
return rendered;
|
|
133
133
|
});
|
|
134
134
|
};
|
|
135
|
-
const REMOTE_INSERT_ROWS_PER_STATEMENT =
|
|
135
|
+
const REMOTE_INSERT_ROWS_PER_STATEMENT = 1000;
|
|
136
136
|
const LOCAL_INSERT_ROWS_PER_STATEMENT = 500;
|
|
137
|
-
const MAX_REMOTE_INSERT_SQL_LENGTH =
|
|
138
|
-
const MAX_REMOTE_EXECUTION_SQL_LENGTH =
|
|
139
|
-
const MIN_REMOTE_INSERT_ROWS_PER_STATEMENT =
|
|
140
|
-
const MAX_REMOTE_INSERT_ROWS_PER_STATEMENT =
|
|
141
|
-
const DEFAULT_REMOTE_TABLE_PARALLELISM =
|
|
137
|
+
const MAX_REMOTE_INSERT_SQL_LENGTH = 300000;
|
|
138
|
+
const MAX_REMOTE_EXECUTION_SQL_LENGTH = 800000;
|
|
139
|
+
const MIN_REMOTE_INSERT_ROWS_PER_STATEMENT = 100;
|
|
140
|
+
const MAX_REMOTE_INSERT_ROWS_PER_STATEMENT = 3000;
|
|
141
|
+
const DEFAULT_REMOTE_TABLE_PARALLELISM = 8;
|
|
142
|
+
const DEFAULT_SOURCE_READ_BATCH_SIZE = 250;
|
|
143
|
+
const MIN_SOURCE_READ_BATCH_SIZE = 50;
|
|
144
|
+
const MAX_SOURCE_READ_BATCH_SIZE = 1500;
|
|
145
|
+
const SOURCE_READ_TARGET_TIME_MS = 5000;
|
|
146
|
+
const SOURCE_CONNECT_RETRY_ATTEMPTS = 3;
|
|
147
|
+
const SOURCE_CONNECT_RETRY_BASE_DELAY_MS = 500;
|
|
142
148
|
const formatDuration = (durationMs) => {
|
|
143
149
|
if (durationMs < 1000) {
|
|
144
150
|
return `${durationMs}ms`;
|
|
@@ -152,6 +158,95 @@ const formatRowsPerSecond = (rows, durationMs) => {
|
|
|
152
158
|
const rate = rows / (durationMs / 1000);
|
|
153
159
|
return `${rate >= 100 ? rate.toFixed(0) : rate.toFixed(2)} rows/s`;
|
|
154
160
|
};
|
|
161
|
+
const waitMs = async (durationMs) => {
|
|
162
|
+
await new Promise((resolve) => {
|
|
163
|
+
const start = Date.now();
|
|
164
|
+
const poll = () => {
|
|
165
|
+
if (Date.now() - start >= durationMs) {
|
|
166
|
+
resolve();
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
globalThis.queueMicrotask(poll);
|
|
170
|
+
};
|
|
171
|
+
poll();
|
|
172
|
+
});
|
|
173
|
+
};
|
|
174
|
+
const calculateOptimalBatchSize = (measuredTimePerRowMs, targetTimePerBatchMs = 5000, minBatchSize = 100, maxBatchSize = 3000) => {
|
|
175
|
+
if (measuredTimePerRowMs <= 0) {
|
|
176
|
+
return minBatchSize;
|
|
177
|
+
}
|
|
178
|
+
const calculatedSize = Math.floor(targetTimePerBatchMs / measuredTimePerRowMs);
|
|
179
|
+
return Math.max(minBatchSize, Math.min(maxBatchSize, calculatedSize));
|
|
180
|
+
};
|
|
181
|
+
const getSourceBatchTuning = (connection, initialBatchSize) => {
|
|
182
|
+
if (connection.sourceBatchTuning !== undefined) {
|
|
183
|
+
return connection.sourceBatchTuning;
|
|
184
|
+
}
|
|
185
|
+
const startingBatchSize = Math.max(MIN_SOURCE_READ_BATCH_SIZE, Math.min(DEFAULT_SOURCE_READ_BATCH_SIZE, initialBatchSize));
|
|
186
|
+
connection.sourceBatchTuning = {
|
|
187
|
+
readBatchSize: startingBatchSize,
|
|
188
|
+
readBatchHistory: [],
|
|
189
|
+
};
|
|
190
|
+
return connection.sourceBatchTuning;
|
|
191
|
+
};
|
|
192
|
+
const isRetryableConnectionError = (error) => {
|
|
193
|
+
const message = getErrorMessage(error).toLowerCase();
|
|
194
|
+
return (message.includes('etimedout') ||
|
|
195
|
+
message.includes('timed out') ||
|
|
196
|
+
message.includes('timeout') ||
|
|
197
|
+
message.includes('econnreset') ||
|
|
198
|
+
message.includes('connection lost') ||
|
|
199
|
+
message.includes('server has gone away') ||
|
|
200
|
+
message.includes('read timeout') ||
|
|
201
|
+
message.includes('write timeout') ||
|
|
202
|
+
message.includes('wait_timeout'));
|
|
203
|
+
};
|
|
204
|
+
const connectWithRetry = async (adapter, label) => {
|
|
205
|
+
let lastError;
|
|
206
|
+
for (let attempt = 0; attempt < SOURCE_CONNECT_RETRY_ATTEMPTS; attempt += 1) {
|
|
207
|
+
try {
|
|
208
|
+
await adapter.connect();
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
lastError = error;
|
|
213
|
+
const retryable = isRetryableConnectionError(error);
|
|
214
|
+
const isLastAttempt = attempt >= SOURCE_CONNECT_RETRY_ATTEMPTS - 1;
|
|
215
|
+
if (!retryable || isLastAttempt) {
|
|
216
|
+
throw error;
|
|
217
|
+
}
|
|
218
|
+
const delayMs = SOURCE_CONNECT_RETRY_BASE_DELAY_MS * 2 ** attempt;
|
|
219
|
+
Logger.warn(`[DataMigrator] ${label} connect failed on attempt ${attempt + 1}/${SOURCE_CONNECT_RETRY_ATTEMPTS}, retrying in ${delayMs}ms: ${getErrorMessage(error)}`);
|
|
220
|
+
await waitMs(delayMs);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
throw lastError;
|
|
224
|
+
};
|
|
225
|
+
const updateSourceBatchTuningAfterSuccess = (connection, rowsRead, durationMs, batchSize) => {
|
|
226
|
+
const tuning = getSourceBatchTuning(connection, batchSize);
|
|
227
|
+
const timePerRow = rowsRead > 0 ? durationMs / rowsRead : durationMs;
|
|
228
|
+
tuning.readBatchHistory.push({ rowsRead, durationMs, timePerRow });
|
|
229
|
+
if (tuning.readBatchHistory.length > 10) {
|
|
230
|
+
tuning.readBatchHistory.shift();
|
|
231
|
+
}
|
|
232
|
+
const avgTimePerRow = tuning.readBatchHistory.reduce((sum, entry) => sum + entry.timePerRow, 0) /
|
|
233
|
+
tuning.readBatchHistory.length;
|
|
234
|
+
const targetTimePerBatchMs = Math.max(1000, Math.min(SOURCE_READ_TARGET_TIME_MS, Math.floor(batchSize * 5)));
|
|
235
|
+
const optimalSize = calculateOptimalBatchSize(avgTimePerRow, targetTimePerBatchMs, MIN_SOURCE_READ_BATCH_SIZE, MAX_SOURCE_READ_BATCH_SIZE);
|
|
236
|
+
if (Math.abs(optimalSize - tuning.readBatchSize) >= 50) {
|
|
237
|
+
tuning.readBatchSize = optimalSize;
|
|
238
|
+
Logger.info(`[DataMigrator] Adaptive source batching: adjusted to ${optimalSize} rows (avg ${avgTimePerRow.toFixed(2)}ms/row)`);
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
const reduceSourceBatchSizeAfterTimeout = (connection, batchSize) => {
|
|
242
|
+
const tuning = getSourceBatchTuning(connection, batchSize);
|
|
243
|
+
const reducedSize = Math.max(MIN_SOURCE_READ_BATCH_SIZE, Math.floor(Math.min(tuning.readBatchSize, batchSize) / 2));
|
|
244
|
+
if (reducedSize < tuning.readBatchSize) {
|
|
245
|
+
tuning.readBatchSize = reducedSize;
|
|
246
|
+
Logger.warn(`[DataMigrator] Adaptive source batching: reduced to ${reducedSize} rows after timeout-like failure`);
|
|
247
|
+
}
|
|
248
|
+
return reducedSize;
|
|
249
|
+
};
|
|
155
250
|
const getRemoteBatchTuning = (connection) => {
|
|
156
251
|
if (connection.remoteBatchTuning !== undefined) {
|
|
157
252
|
return connection.remoteBatchTuning;
|
|
@@ -185,18 +280,155 @@ const adjustRemoteBatchTuning = (connection, executedRows, durationMs, sqlLength
|
|
|
185
280
|
const tuning = getRemoteBatchTuning(connection);
|
|
186
281
|
const previousRowsPerStatement = tuning.rowsPerStatement;
|
|
187
282
|
const nearSqlLimit = sqlLength >= Math.floor(tuning.maxExecutionSqlLength * 0.9);
|
|
188
|
-
if (durationMs <=
|
|
283
|
+
if (durationMs <= 2500 &&
|
|
189
284
|
executedRows >= Math.floor(previousRowsPerStatement * 0.8) &&
|
|
190
285
|
!nearSqlLimit) {
|
|
191
|
-
tuning.rowsPerStatement = Math.min(MAX_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.max(previousRowsPerStatement +
|
|
286
|
+
tuning.rowsPerStatement = Math.min(MAX_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.max(previousRowsPerStatement + 50, Math.floor(previousRowsPerStatement * 1.3)));
|
|
192
287
|
}
|
|
193
|
-
else if (durationMs >=
|
|
194
|
-
tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement -
|
|
288
|
+
else if (durationMs >= 10000 || nearSqlLimit) {
|
|
289
|
+
tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement - 50, Math.floor(previousRowsPerStatement * 0.8)));
|
|
195
290
|
}
|
|
196
291
|
if (tuning.rowsPerStatement !== previousRowsPerStatement) {
|
|
197
292
|
Logger.info(`[DataMigrator] Adaptive remote batching: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after ${executedRows} rows in ${formatDuration(durationMs)}`);
|
|
198
293
|
}
|
|
199
294
|
};
|
|
295
|
+
const reduceRemoteBatchTuningAfterFailure = (connection, sqlLength) => {
|
|
296
|
+
if (connection.type !== 'd1-remote') {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const tuning = getRemoteBatchTuning(connection);
|
|
300
|
+
const previousRowsPerStatement = tuning.rowsPerStatement;
|
|
301
|
+
const nearSqlLimit = sqlLength >= Math.floor(tuning.maxExecutionSqlLength * 0.8);
|
|
302
|
+
if (!nearSqlLimit && previousRowsPerStatement <= MIN_REMOTE_INSERT_ROWS_PER_STATEMENT) {
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement - 50, Math.floor(previousRowsPerStatement * 0.5)));
|
|
306
|
+
if (tuning.rowsPerStatement !== previousRowsPerStatement) {
|
|
307
|
+
Logger.warn(`[DataMigrator] Adaptive remote batching: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after failed remote insert`);
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
const logTableMigrationProgress = (table, rowsMigrated, totalRows, batchSize) => {
|
|
311
|
+
if (totalRows <= 10000 || rowsMigrated % (batchSize * 10) !== 0) {
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
const normalizedTotalRows = Math.max(totalRows, rowsMigrated);
|
|
315
|
+
const percentage = Math.round((rowsMigrated / normalizedTotalRows) * 100);
|
|
316
|
+
Logger.info(`Table ${table.name}: ${rowsMigrated}/${normalizedTotalRows} (${percentage}%)`);
|
|
317
|
+
};
|
|
318
|
+
const getTableChunkBatchSize = (sourceConnection, retryBatchSize) => {
|
|
319
|
+
if (sourceConnection.driver !== 'mysql') {
|
|
320
|
+
return retryBatchSize;
|
|
321
|
+
}
|
|
322
|
+
return getSourceBatchTuning(sourceConnection, retryBatchSize).readBatchSize;
|
|
323
|
+
};
|
|
324
|
+
const applyTableChunkResult = (state, result, initialBatchSize, table, totalRows) => {
|
|
325
|
+
const nextState = {
|
|
326
|
+
rowsMigrated: state.rowsMigrated + result.rowsMigrated,
|
|
327
|
+
offset: result.nextOffset,
|
|
328
|
+
currentBatchSize: result.nextBatchSize,
|
|
329
|
+
retryBatchSize: result.nextBatchSize,
|
|
330
|
+
};
|
|
331
|
+
logTableMigrationProgress(table, nextState.rowsMigrated, totalRows, nextState.currentBatchSize);
|
|
332
|
+
if (!result.continueProcessing &&
|
|
333
|
+
result.rowsMigrated === 0 &&
|
|
334
|
+
result.nextBatchSize < initialBatchSize) {
|
|
335
|
+
return nextState;
|
|
336
|
+
}
|
|
337
|
+
return nextState;
|
|
338
|
+
};
|
|
339
|
+
const verifyChunkInsertCount = (tableName, offset, expectedRows, insertedRows) => {
|
|
340
|
+
if (insertedRows === expectedRows) {
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
const verificationError = DataMigrator.createChunkVerificationError(tableName, offset, expectedRows, insertedRows);
|
|
344
|
+
throw ErrorFactory.createValidationError(`Chunk insert mismatch on ${tableName}`, verificationError);
|
|
345
|
+
};
|
|
346
|
+
const logChunkProcessingSuccess = (tableName, offset, insertedRows, chunkDurationMs) => {
|
|
347
|
+
Logger.info(`[DataMigrator] Chunk ${tableName} offset=${offset} rows=${insertedRows} duration=${formatDuration(chunkDurationMs)} rate=${formatRowsPerSecond(insertedRows, chunkDurationMs)}`);
|
|
348
|
+
};
|
|
349
|
+
const executeChunkRows = async (context) => {
|
|
350
|
+
const chunk = await DataMigrator.readDataChunk(context.sourceConnection, context.table.name, context.offset, context.batchSize);
|
|
351
|
+
if (chunk.length === 0) {
|
|
352
|
+
return null;
|
|
353
|
+
}
|
|
354
|
+
const transformedChunk = await DataMigrator.transformData(chunk, context.table.name);
|
|
355
|
+
const insertedRows = await DataMigrator.insertData(context.targetConnection, context.table.name, transformedChunk);
|
|
356
|
+
verifyChunkInsertCount(context.table.name, context.offset, chunk.length, insertedRows);
|
|
357
|
+
return {
|
|
358
|
+
chunkLength: chunk.length,
|
|
359
|
+
insertedRows,
|
|
360
|
+
};
|
|
361
|
+
};
|
|
362
|
+
const createEmptyChunkResult = (offset, batchSize) => {
|
|
363
|
+
return {
|
|
364
|
+
rowsMigrated: 0,
|
|
365
|
+
nextOffset: offset,
|
|
366
|
+
nextBatchSize: batchSize,
|
|
367
|
+
continueProcessing: false,
|
|
368
|
+
};
|
|
369
|
+
};
|
|
370
|
+
const buildChunkProcessingSuccessResult = async (table, sourceConnection, targetConnection, offset, batchSize) => {
|
|
371
|
+
const chunkStartTime = Date.now();
|
|
372
|
+
const executionResult = await executeChunkRows({
|
|
373
|
+
table,
|
|
374
|
+
sourceConnection,
|
|
375
|
+
targetConnection,
|
|
376
|
+
offset,
|
|
377
|
+
batchSize,
|
|
378
|
+
});
|
|
379
|
+
if (executionResult === null) {
|
|
380
|
+
return createEmptyChunkResult(offset, batchSize);
|
|
381
|
+
}
|
|
382
|
+
const { chunkLength, insertedRows } = executionResult;
|
|
383
|
+
const chunkDurationMs = Date.now() - chunkStartTime;
|
|
384
|
+
if (sourceConnection.driver === 'mysql') {
|
|
385
|
+
updateSourceBatchTuningAfterSuccess(sourceConnection, insertedRows, chunkDurationMs, batchSize);
|
|
386
|
+
}
|
|
387
|
+
logChunkProcessingSuccess(table.name, offset, insertedRows, chunkDurationMs);
|
|
388
|
+
if (chunkLength < batchSize) {
|
|
389
|
+
return {
|
|
390
|
+
rowsMigrated: insertedRows,
|
|
391
|
+
nextOffset: offset + chunkLength,
|
|
392
|
+
nextBatchSize: batchSize,
|
|
393
|
+
continueProcessing: false,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
return {
|
|
397
|
+
rowsMigrated: insertedRows,
|
|
398
|
+
nextOffset: offset + batchSize,
|
|
399
|
+
nextBatchSize: batchSize,
|
|
400
|
+
continueProcessing: true,
|
|
401
|
+
};
|
|
402
|
+
};
|
|
403
|
+
const buildChunkProcessingFailureResult = (sourceConnection, offset, batchSize, error, errors) => {
|
|
404
|
+
const errorMessage = getErrorMessage(error);
|
|
405
|
+
const errorMsg = `Chunk processing failed at offset ${offset}: ${errorMessage}`;
|
|
406
|
+
Logger.warn(errorMsg);
|
|
407
|
+
errors.push(errorMsg);
|
|
408
|
+
if (sourceConnection.driver === 'mysql' && isRetryableConnectionError(error)) {
|
|
409
|
+
const reducedBatchSize = reduceSourceBatchSizeAfterTimeout(sourceConnection, batchSize);
|
|
410
|
+
return {
|
|
411
|
+
rowsMigrated: 0,
|
|
412
|
+
nextOffset: offset,
|
|
413
|
+
nextBatchSize: reducedBatchSize,
|
|
414
|
+
continueProcessing: reducedBatchSize >= batchSize,
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
return {
|
|
418
|
+
rowsMigrated: 0,
|
|
419
|
+
nextOffset: offset + batchSize,
|
|
420
|
+
nextBatchSize: batchSize,
|
|
421
|
+
continueProcessing: true,
|
|
422
|
+
};
|
|
423
|
+
};
|
|
424
|
+
const processTableChunk = async (table, sourceConnection, targetConnection, offset, batchSize, errors) => {
|
|
425
|
+
try {
|
|
426
|
+
return await buildChunkProcessingSuccessResult(table, sourceConnection, targetConnection, offset, batchSize);
|
|
427
|
+
}
|
|
428
|
+
catch (error) {
|
|
429
|
+
return buildChunkProcessingFailureResult(sourceConnection, offset, batchSize, error, errors);
|
|
430
|
+
}
|
|
431
|
+
};
|
|
200
432
|
const getTableDependencies = (table) => {
|
|
201
433
|
if (!Array.isArray(table.dependsOn)) {
|
|
202
434
|
return [];
|
|
@@ -426,9 +658,6 @@ const logDetailedError = (label, error) => {
|
|
|
426
658
|
if (driverDetails !== undefined) {
|
|
427
659
|
Logger.error(`${label} driver details: ${driverDetails}`);
|
|
428
660
|
}
|
|
429
|
-
if (error instanceof Error && typeof error.stack === 'string' && error.stack.trim() !== '') {
|
|
430
|
-
Logger.error(`${label} stack: ${error.stack}`);
|
|
431
|
-
}
|
|
432
661
|
const cause = getErrorCause(error);
|
|
433
662
|
if (cause !== undefined) {
|
|
434
663
|
logDetailedError(`${label} cause`, cause);
|
|
@@ -487,6 +716,14 @@ const createSourceAdapter = (config) => {
|
|
|
487
716
|
username: connectionDetails.username,
|
|
488
717
|
password: connectionDetails.password,
|
|
489
718
|
ssl: config.sourceSsl,
|
|
719
|
+
socketTimeoutMs: config.sourceSocketTimeoutMs,
|
|
720
|
+
connectTimeoutMs: Math.max(config.sourceSocketTimeoutMs ?? 30000, 60000),
|
|
721
|
+
acquireTimeoutMs: Math.max(config.sourceSocketTimeoutMs ?? 30000, 60000),
|
|
722
|
+
enableKeepAlive: true,
|
|
723
|
+
keepAliveInitialDelayMs: 0,
|
|
724
|
+
waitTimeoutSeconds: config.sourceWaitTimeoutSeconds,
|
|
725
|
+
netReadTimeoutSeconds: config.sourceNetReadTimeoutSeconds,
|
|
726
|
+
netWriteTimeoutSeconds: config.sourceNetWriteTimeoutSeconds,
|
|
490
727
|
});
|
|
491
728
|
}
|
|
492
729
|
case 'postgresql': {
|
|
@@ -599,7 +836,6 @@ export const DataMigrator = Object.freeze({
|
|
|
599
836
|
const errorChain = getErrorChainMessages(error);
|
|
600
837
|
Logger.error(`Data migration failed: ${errorChain.join(' -> ')}`);
|
|
601
838
|
logDetailedError('Data migration failure', error);
|
|
602
|
-
Logger.error('Data migration failure details:', error);
|
|
603
839
|
throw error;
|
|
604
840
|
}
|
|
605
841
|
finally {
|
|
@@ -614,13 +850,12 @@ export const DataMigrator = Object.freeze({
|
|
|
614
850
|
Logger.info(`Connecting to ${config.sourceDriver} database...`);
|
|
615
851
|
const adapter = createSourceAdapter(config);
|
|
616
852
|
try {
|
|
617
|
-
await adapter.
|
|
853
|
+
await connectWithRetry(adapter, config.sourceDriver);
|
|
618
854
|
}
|
|
619
855
|
catch (error) {
|
|
620
856
|
const errorChain = getErrorChainMessages(error);
|
|
621
857
|
Logger.error(`Source database connection failed: ${errorChain.join(' -> ')}`);
|
|
622
858
|
logDetailedError('Source database connection failure', error);
|
|
623
|
-
Logger.error('Source database connection failure details:', error);
|
|
624
859
|
throw error;
|
|
625
860
|
}
|
|
626
861
|
const sourceConnection = {
|
|
@@ -632,6 +867,9 @@ export const DataMigrator = Object.freeze({
|
|
|
632
867
|
adapter,
|
|
633
868
|
};
|
|
634
869
|
Logger.info('✓ Source database connected');
|
|
870
|
+
if (config.sourceDriver === 'mysql') {
|
|
871
|
+
sourceConnection.sourceBatchTuning = getSourceBatchTuning(sourceConnection, config.batchSize ?? DEFAULT_SOURCE_READ_BATCH_SIZE);
|
|
872
|
+
}
|
|
635
873
|
return sourceConnection;
|
|
636
874
|
},
|
|
637
875
|
/**
|
|
@@ -677,6 +915,7 @@ export const DataMigrator = Object.freeze({
|
|
|
677
915
|
if (!targetConnection.adapter) {
|
|
678
916
|
throw ErrorFactory.createConnectionError('No target adapter available for D1 schema preparation');
|
|
679
917
|
}
|
|
918
|
+
const adapter = targetConnection.adapter;
|
|
680
919
|
Logger.info('Preparing target D1 schema...');
|
|
681
920
|
const sourceSchema = await SchemaAnalyzer.analyzeSchema({
|
|
682
921
|
driver: sourceConnection.driver,
|
|
@@ -686,16 +925,42 @@ export const DataMigrator = Object.freeze({
|
|
|
686
925
|
});
|
|
687
926
|
const d1Schema = SchemaBuilder.buildD1Schema(sourceSchema.tables, config.sourceDriver);
|
|
688
927
|
SchemaBuilder.assertValidSchema(d1Schema);
|
|
928
|
+
const schemaStatements = [];
|
|
929
|
+
const maxBatchSqlLength = targetConnection.type === 'd1-remote'
|
|
930
|
+
? Math.max(0, MAX_REMOTE_EXECUTION_SQL_LENGTH - 2000)
|
|
931
|
+
: Number.POSITIVE_INFINITY;
|
|
932
|
+
const executeSchemaBatch = async () => {
|
|
933
|
+
if (schemaStatements.length === 0) {
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
const batchSql = `${schemaStatements.join(';\n')};`;
|
|
937
|
+
await adapter.query(batchSql, []);
|
|
938
|
+
schemaStatements.length = 0;
|
|
939
|
+
};
|
|
940
|
+
const pushSchemaStatement = async (sql) => {
|
|
941
|
+
const trimmedSql = sql.trim();
|
|
942
|
+
const nextLength = schemaStatements.length === 0
|
|
943
|
+
? trimmedSql.length
|
|
944
|
+
: schemaStatements.join(';\n').length + 2 + trimmedSql.length;
|
|
945
|
+
if (targetConnection.type === 'd1-remote' && nextLength > maxBatchSqlLength) {
|
|
946
|
+
await executeSchemaBatch();
|
|
947
|
+
}
|
|
948
|
+
schemaStatements.push(trimmedSql);
|
|
949
|
+
};
|
|
689
950
|
for (const table of d1Schema) {
|
|
690
951
|
const createSQL = SchemaBuilder.generateCreateTableSQL(table).replace(/^CREATE TABLE\s+/i, 'CREATE TABLE IF NOT EXISTS ');
|
|
691
|
-
await
|
|
952
|
+
await pushSchemaStatement(createSQL);
|
|
692
953
|
const indexSQL = SchemaBuilder.generateIndexSQL(table).map((sql) => sql
|
|
693
954
|
.replace(/^CREATE\s+UNIQUE\s+INDEX\s+/i, 'CREATE UNIQUE INDEX IF NOT EXISTS ')
|
|
694
955
|
.replace(/^CREATE\s+INDEX\s+/i, 'CREATE INDEX IF NOT EXISTS '));
|
|
695
956
|
for (const sql of indexSQL) {
|
|
696
|
-
await
|
|
957
|
+
await pushSchemaStatement(sql);
|
|
958
|
+
}
|
|
959
|
+
if (targetConnection.type !== 'd1-remote') {
|
|
960
|
+
await executeSchemaBatch();
|
|
697
961
|
}
|
|
698
962
|
}
|
|
963
|
+
await executeSchemaBatch();
|
|
699
964
|
Logger.info(`✓ Target schema prepared for ${d1Schema.length} tables`);
|
|
700
965
|
},
|
|
701
966
|
/**
|
|
@@ -768,36 +1033,22 @@ export const DataMigrator = Object.freeze({
|
|
|
768
1033
|
}
|
|
769
1034
|
},
|
|
770
1035
|
async processTableChunks(table, sourceConnection, targetConnection, totalRows, batchSize, startOffset, errors) {
|
|
771
|
-
let
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
}
|
|
785
|
-
rowsMigrated += insertedRows;
|
|
786
|
-
const chunkDurationMs = Date.now() - chunkStartTime;
|
|
787
|
-
Logger.info(`[DataMigrator] Chunk ${table.name} offset=${offset} rows=${insertedRows} duration=${formatDuration(chunkDurationMs)} rate=${formatRowsPerSecond(insertedRows, chunkDurationMs)}`);
|
|
788
|
-
if (totalRows > 10000 && rowsMigrated % (batchSize * 10) === 0) {
|
|
789
|
-
const normalizedTotalRows = Math.max(totalRows, rowsMigrated);
|
|
790
|
-
const percentage = Math.round((rowsMigrated / normalizedTotalRows) * 100);
|
|
791
|
-
Logger.info(`Table ${table.name}: ${rowsMigrated}/${normalizedTotalRows} (${percentage}%)`);
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
catch (error) {
|
|
795
|
-
const errorMsg = `Chunk processing failed at offset ${offset}: ${error}`;
|
|
796
|
-
Logger.error(errorMsg);
|
|
797
|
-
errors.push(errorMsg);
|
|
1036
|
+
let state = {
|
|
1037
|
+
rowsMigrated: 0,
|
|
1038
|
+
offset: startOffset,
|
|
1039
|
+
currentBatchSize: batchSize,
|
|
1040
|
+
retryBatchSize: batchSize,
|
|
1041
|
+
};
|
|
1042
|
+
while (state.offset < totalRows) {
|
|
1043
|
+
state.currentBatchSize = getTableChunkBatchSize(sourceConnection, state.retryBatchSize);
|
|
1044
|
+
const result = await processTableChunk(table, sourceConnection, targetConnection, state.offset, state.currentBatchSize, errors);
|
|
1045
|
+
state = applyTableChunkResult(state, result, batchSize, table, totalRows);
|
|
1046
|
+
if (!result.continueProcessing &&
|
|
1047
|
+
!(result.rowsMigrated === 0 && result.nextBatchSize < batchSize)) {
|
|
1048
|
+
break;
|
|
798
1049
|
}
|
|
799
1050
|
}
|
|
800
|
-
return rowsMigrated;
|
|
1051
|
+
return state.rowsMigrated;
|
|
801
1052
|
},
|
|
802
1053
|
/**
|
|
803
1054
|
* Read data chunk from source database
|
|
@@ -807,12 +1058,21 @@ export const DataMigrator = Object.freeze({
|
|
|
807
1058
|
return [];
|
|
808
1059
|
try {
|
|
809
1060
|
const selectSql = DataMigrator.buildSelectChunkSQL(sourceConnection.driver, tableName);
|
|
1061
|
+
const startTime = Date.now();
|
|
810
1062
|
const result = await sourceConnection.adapter.query(`${selectSql} LIMIT ${batchSize} OFFSET ${offset}`, []);
|
|
1063
|
+
const durationMs = Date.now() - startTime;
|
|
1064
|
+
if (sourceConnection.driver === 'mysql') {
|
|
1065
|
+
updateSourceBatchTuningAfterSuccess(sourceConnection, result.rows.length, durationMs, batchSize);
|
|
1066
|
+
}
|
|
811
1067
|
return result.rows || [];
|
|
812
1068
|
}
|
|
813
1069
|
catch (error) {
|
|
814
|
-
|
|
815
|
-
|
|
1070
|
+
const errorMessage = getErrorMessage(error);
|
|
1071
|
+
if (sourceConnection.driver === 'mysql' && isRetryableConnectionError(error)) {
|
|
1072
|
+
reduceSourceBatchSizeAfterTimeout(sourceConnection, batchSize);
|
|
1073
|
+
}
|
|
1074
|
+
Logger.warn(`Chunk read failed: ${errorMessage}`);
|
|
1075
|
+
throw error;
|
|
816
1076
|
}
|
|
817
1077
|
},
|
|
818
1078
|
/**
|
|
@@ -873,11 +1133,16 @@ export const DataMigrator = Object.freeze({
|
|
|
873
1133
|
insertedRows += affectedRows;
|
|
874
1134
|
adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, statement.sql.length);
|
|
875
1135
|
}
|
|
876
|
-
catch
|
|
1136
|
+
catch {
|
|
1137
|
+
if (targetConnection.type === 'd1-remote' && data.length > 1 && statement.rowCount > 1) {
|
|
1138
|
+
reduceRemoteBatchTuningAfterFailure(targetConnection, statement.sql.length);
|
|
1139
|
+
const midpoint = Math.max(1, Math.floor(data.length / 2));
|
|
1140
|
+
const left = await DataMigrator.insertData(targetConnection, tableName, data.slice(0, midpoint));
|
|
1141
|
+
const right = await DataMigrator.insertData(targetConnection, tableName, data.slice(midpoint));
|
|
1142
|
+
return left + right;
|
|
1143
|
+
}
|
|
877
1144
|
throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
|
|
878
|
-
sql: statement.sql,
|
|
879
1145
|
rowCount: statement.rowCount,
|
|
880
|
-
cause: error,
|
|
881
1146
|
});
|
|
882
1147
|
}
|
|
883
1148
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,UAAU,CAAC;AAOxE,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAouBF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,
|
|
1
|
+
{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,UAAU,CAAC;AAOxE,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAouBF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,iBA6H/B,CAAC;AAEH;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAetE;AAED;;GAEG;AACH,iBAAe,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAmGxE;AAED;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGtE;AAED;;GAEG;AACH,iBAAS,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrF;AAGD,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC"}
|
|
@@ -588,6 +588,16 @@ export const MigrateToD1Command = BaseCommand.create({
|
|
|
588
588
|
// Execute data migration
|
|
589
589
|
Logger.info('Starting data migration...');
|
|
590
590
|
const migrationProgress = await DataMigrator.migrateData(config);
|
|
591
|
+
if (migrationProgress.status === 'failed') {
|
|
592
|
+
const errorDetails = Object.values(migrationProgress.errors).filter((error) => {
|
|
593
|
+
return typeof error === 'string' && error.trim().length > 0;
|
|
594
|
+
});
|
|
595
|
+
Logger.error(`Migration finished with failures: ${migrationProgress.processedRows} rows migrated`);
|
|
596
|
+
if (errorDetails.length > 0) {
|
|
597
|
+
Logger.error(`Migration error details: ${errorDetails.join(' | ')}`);
|
|
598
|
+
}
|
|
599
|
+
throw ErrorFactory.createValidationError(`Migration failed after migrating ${migrationProgress.processedRows} rows`);
|
|
600
|
+
}
|
|
591
601
|
Logger.info(`Migration completed: ${migrationProgress.processedRows} rows migrated`);
|
|
592
602
|
Logger.info('D1 migration completed successfully');
|
|
593
603
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -17,6 +17,10 @@ export interface MigrationConfig {
|
|
|
17
17
|
dryRun?: boolean;
|
|
18
18
|
interactive?: boolean;
|
|
19
19
|
migrationId?: string;
|
|
20
|
+
sourceSocketTimeoutMs?: number;
|
|
21
|
+
sourceWaitTimeoutSeconds?: number;
|
|
22
|
+
sourceNetReadTimeoutSeconds?: number;
|
|
23
|
+
sourceNetWriteTimeoutSeconds?: number;
|
|
20
24
|
}
|
|
21
25
|
export interface MigrationState {
|
|
22
26
|
id: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,MAAM,oBAAoB,GAAG,OAAO,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEnF,MAAM,MAAM,sBAAsB,GAAG,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC;AAEjE,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,oBAAoB,CAAC;IACnC,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,IAAI,GAAG,WAAW,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,MAAM,oBAAoB,GAAG,OAAO,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEnF,MAAM,MAAM,sBAAsB,GAAG,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC;AAEjE,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,oBAAoB,CAAC;IACnC,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,IAAI,GAAG,WAAW,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;IAClE,SAAS,EAAE,IAAI,CAAC;IAChB,cAAc,CAAC,EAAE,IAAI,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,aAAa,EAAE,iBAAiB,EAAE,CAAC;IACnC,WAAW,EAAE,eAAe,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,YAAY,GAAG,aAAa,GAAG,cAAc,CAAC;CACrD;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,aAAa,GAAG,aAAa,GAAG,QAAQ,GAAG,OAAO,GAAG,UAAU,CAAC;IACtE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,UAAU,CAAC;IAC/C,QAAQ,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,UAAU,CAAC;CAChD;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,kBAAkB,GAAG,iBAAiB,GAAG,yBAAyB,CAAC;IACzE,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,gBAAgB,GAAG,oBAAoB,GAAG,eAAe,CAAC;IAChE,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC1D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,OAAO,CAAC,EAAE,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,UAAU,EAAE,MAAM,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,kBAAkB,KAAK,IAAI,CAAC;CAC9D,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,kBAAkB,EAAE,kBAAkB,CAAC;CACxC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/d1-migrator",
|
|
3
|
-
"version": "1.9.
|
|
3
|
+
"version": "1.9.6",
|
|
4
4
|
"description": "Resumable database migration toolkit for moving data to Cloudflare D1 with ZinTrust.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"type": "module",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"@zintrust/db-d1": "^1.8.0",
|
|
44
|
-
"@zintrust/db-mysql": "^
|
|
44
|
+
"@zintrust/db-mysql": "^2.0.1",
|
|
45
45
|
"@zintrust/db-postgres": "^1.8.0",
|
|
46
46
|
"@zintrust/db-sqlite": "^1.8.0",
|
|
47
47
|
"@zintrust/db-sqlserver": "^1.8.0"
|