@zintrust/d1-migrator 2.0.0 → 2.0.2

package/dist/cli/DataMigrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAQD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAqnCF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IA+EhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BAoD7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IAiClB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IA+DlB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
+{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAyrCF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IA+EhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BA0E7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA8BlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAyClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}

package/dist/cli/DataMigrator.js

@@ -4,6 +4,8 @@
  * Handles the actual data migration between databases
  */
 import { ErrorFactory, LocalD1Resolver, Logger, WranglerD1 } from '@zintrust/core';
+import { appendFileSync, mkdirSync } from 'node:fs';
+import * as path from 'node:path';
 import { MySQLAdapter } from '@zintrust/db-mysql';
 import { PostgreSQLAdapter } from '@zintrust/db-postgres';
 import { SQLiteAdapter } from '@zintrust/db-sqlite';
@@ -134,17 +136,17 @@ const bindSqlParameters = (sql, parameters) => {
 };
 const REMOTE_INSERT_ROWS_PER_STATEMENT = 1000;
 const LOCAL_INSERT_ROWS_PER_STATEMENT = 500;
-const MAX_REMOTE_INSERT_SQL_LENGTH = 300000;
-const MAX_REMOTE_EXECUTION_SQL_LENGTH = 800000;
-const MIN_REMOTE_INSERT_ROWS_PER_STATEMENT = 100;
-const MAX_REMOTE_INSERT_ROWS_PER_STATEMENT = 3000;
-const DEFAULT_REMOTE_TABLE_PARALLELISM = 8;
-const DEFAULT_SOURCE_READ_BATCH_SIZE = 250;
-const MIN_SOURCE_READ_BATCH_SIZE = 50;
-const MAX_SOURCE_READ_BATCH_SIZE = 1500;
-const SOURCE_READ_TARGET_TIME_MS = 5000;
+const MAX_REMOTE_INSERT_ROWS_PER_STATEMENT = 2000;
+const MAX_REMOTE_INSERT_SQL_LENGTH = 95 * 1024;
+const MAX_REMOTE_EXECUTION_SQL_LENGTH = MAX_REMOTE_INSERT_SQL_LENGTH;
+const MIN_REMOTE_INSERT_ROWS_PER_STATEMENT = 1;
+const DEFAULT_REMOTE_TABLE_PARALLELISM = 1;
+const DEFAULT_SOURCE_READ_BATCH_SIZE = 1000;
+const MIN_SOURCE_READ_BATCH_SIZE = 1;
 const SOURCE_CONNECT_RETRY_ATTEMPTS = 3;
 const SOURCE_CONNECT_RETRY_BASE_DELAY_MS = 500;
+const FAILED_TABLE_REPORT_PATH = path.join('logs', 'd1-migration-failed-report.log');
+const REMOTE_SQL_GROW_THRESHOLD_RATIO = 0.6;
 const formatDuration = (durationMs) => {
     if (durationMs < 1000) {
         return `${durationMs}ms`;
@@ -158,6 +160,9 @@ const formatRowsPerSecond = (rows, durationMs) => {
     const rate = rows / (durationMs / 1000);
     return `${rate >= 100 ? rate.toFixed(0) : rate.toFixed(2)} rows/s`;
 };
+const getSqlByteLength = (value) => {
+    return Buffer.byteLength(value, 'utf8');
+};
 const waitMs = async (durationMs) => {
     await new Promise((resolve) => {
         const start = Date.now();
@@ -171,24 +176,55 @@ const waitMs = async (durationMs) => {
         poll();
     });
 };
-const calculateOptimalBatchSize = (measuredTimePerRowMs, targetTimePerBatchMs = 5000, minBatchSize = 100, maxBatchSize = 3000) => {
-    if (measuredTimePerRowMs <= 0) {
-        return minBatchSize;
-    }
-    const calculatedSize = Math.floor(targetTimePerBatchMs / measuredTimePerRowMs);
-    return Math.max(minBatchSize, Math.min(maxBatchSize, calculatedSize));
+const normalizeSourceBatchSize = (requestedBatchSize) => {
+    return Math.max(MIN_SOURCE_READ_BATCH_SIZE, requestedBatchSize);
+};
+const createSourceBatchTuning = (initialBatchSize) => {
+    return {
+        readBatchSize: normalizeSourceBatchSize(initialBatchSize),
+        readBatchHistory: [],
+    };
+};
+const createRemoteBatchTuning = () => {
+    return {
+        rowsPerStatement: REMOTE_INSERT_ROWS_PER_STATEMENT,
+        maxStatementSqlLength: MAX_REMOTE_INSERT_SQL_LENGTH,
+        maxExecutionSqlLength: MAX_REMOTE_EXECUTION_SQL_LENGTH,
+    };
+};
+const sanitizeFailureReason = (reason) => {
+    return reason.replaceAll(/\s+/g, ' ').trim();
+};
+const appendFailedTableReport = (params) => {
+    const reportPath = path.join(process.cwd(), FAILED_TABLE_REPORT_PATH);
+    const reportDir = path.dirname(reportPath);
+    const reason = sanitizeFailureReason(params.errors.at(-1) ?? 'Unknown migration failure');
+    const entry = JSON.stringify({
+        timestamp: new Date().toISOString(),
+        migrationId: params.migrationId,
+        targetDatabase: params.targetDatabase,
+        tableName: params.tableName,
+        rowsMigrated: params.rowsMigrated,
+        totalRows: params.totalRows,
+        reason,
+    });
+    mkdirSync(reportDir, { recursive: true });
+    appendFileSync(reportPath, `${entry}\n`, 'utf8');
+    Logger.warn(`[DataMigrator] Skipped remaining rows for ${params.tableName}. Failure recorded in ${FAILED_TABLE_REPORT_PATH}`);
 };
 const getSourceBatchTuning = (connection, initialBatchSize) => {
     if (connection.sourceBatchTuning !== undefined) {
         return connection.sourceBatchTuning;
     }
-    const startingBatchSize = Math.max(MIN_SOURCE_READ_BATCH_SIZE, Math.min(DEFAULT_SOURCE_READ_BATCH_SIZE, initialBatchSize));
-    connection.sourceBatchTuning = {
-        readBatchSize: startingBatchSize,
-        readBatchHistory: [],
-    };
+    connection.sourceBatchTuning = createSourceBatchTuning(initialBatchSize);
     return connection.sourceBatchTuning;
 };
+const resetSourceBatchTuning = (connection, initialBatchSize) => {
+    if (connection.driver !== 'mysql') {
+        return;
+    }
+    connection.sourceBatchTuning = createSourceBatchTuning(initialBatchSize);
+};
 const isRetryableConnectionError = (error) => {
     const message = getErrorMessage(error).toLowerCase();
     return (message.includes('etimedout') ||
@@ -229,35 +265,31 @@ const updateSourceBatchTuningAfterSuccess = (connection, rowsRead, durationMs, b
     if (tuning.readBatchHistory.length > 10) {
         tuning.readBatchHistory.shift();
     }
-    const avgTimePerRow = tuning.readBatchHistory.reduce((sum, entry) => sum + entry.timePerRow, 0) /
-        tuning.readBatchHistory.length;
-    const targetTimePerBatchMs = Math.max(1000, Math.min(SOURCE_READ_TARGET_TIME_MS, Math.floor(batchSize * 5)));
-    const optimalSize = calculateOptimalBatchSize(avgTimePerRow, targetTimePerBatchMs, MIN_SOURCE_READ_BATCH_SIZE, MAX_SOURCE_READ_BATCH_SIZE);
-    if (Math.abs(optimalSize - tuning.readBatchSize) >= 50) {
-        tuning.readBatchSize = optimalSize;
-        Logger.info(`[DataMigrator] Adaptive source batching: adjusted to ${optimalSize} rows (avg ${avgTimePerRow.toFixed(2)}ms/row)`);
-    }
 };
 const reduceSourceBatchSizeAfterTimeout = (connection, batchSize) => {
     const tuning = getSourceBatchTuning(connection, batchSize);
-    const reducedSize = Math.max(MIN_SOURCE_READ_BATCH_SIZE, Math.floor(Math.min(tuning.readBatchSize, batchSize) / 2));
-    if (reducedSize < tuning.readBatchSize) {
-        tuning.readBatchSize = reducedSize;
-        Logger.warn(`[DataMigrator] Adaptive source batching: reduced to ${reducedSize} rows after timeout-like failure`);
+    const currentSize = Math.min(tuning.readBatchSize, batchSize);
+    if (currentSize <= MIN_SOURCE_READ_BATCH_SIZE) {
+        return 0;
     }
+    const reducedSize = Math.max(MIN_SOURCE_READ_BATCH_SIZE, Math.floor(currentSize / 2));
+    tuning.readBatchSize = reducedSize;
+    Logger.warn(`[DataMigrator] Source batch retry: ${currentSize} -> ${reducedSize} rows after retryable read failure`);
     return reducedSize;
 };
 const getRemoteBatchTuning = (connection) => {
     if (connection.remoteBatchTuning !== undefined) {
         return connection.remoteBatchTuning;
     }
-    connection.remoteBatchTuning = {
-        rowsPerStatement: REMOTE_INSERT_ROWS_PER_STATEMENT,
-        maxStatementSqlLength: MAX_REMOTE_INSERT_SQL_LENGTH,
-        maxExecutionSqlLength: MAX_REMOTE_EXECUTION_SQL_LENGTH,
-    };
+    connection.remoteBatchTuning = createRemoteBatchTuning();
     return connection.remoteBatchTuning;
 };
+const resetRemoteBatchTuning = (connection) => {
+    if (connection.type !== 'd1-remote') {
+        return;
+    }
+    connection.remoteBatchTuning = createRemoteBatchTuning();
+};
 const getInsertBatchSettings = (connection) => {
     if (connection.type === 'd1-remote') {
         const tuning = getRemoteBatchTuning(connection);
@@ -279,33 +311,32 @@ const adjustRemoteBatchTuning = (connection, executedRows, durationMs, sqlLength
     }
     const tuning = getRemoteBatchTuning(connection);
     const previousRowsPerStatement = tuning.rowsPerStatement;
-    const nearSqlLimit = sqlLength >= Math.floor(tuning.maxExecutionSqlLength * 0.9);
-    if (durationMs <= 2500 &&
-        executedRows >= Math.floor(previousRowsPerStatement * 0.8) &&
-        !nearSqlLimit) {
-        tuning.rowsPerStatement = Math.min(MAX_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.max(previousRowsPerStatement + 50, Math.floor(previousRowsPerStatement * 1.3)));
-    }
-    else if (durationMs >= 10000 || nearSqlLimit) {
-        tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement - 50, Math.floor(previousRowsPerStatement * 0.8)));
+    const growthThreshold = Math.floor(tuning.maxStatementSqlLength * REMOTE_SQL_GROW_THRESHOLD_RATIO);
+    if (previousRowsPerStatement >= MAX_REMOTE_INSERT_ROWS_PER_STATEMENT ||
+        executedRows < previousRowsPerStatement ||
+        sqlLength > growthThreshold ||
+        durationMs > 10000) {
+        return;
     }
+    tuning.rowsPerStatement = Math.min(MAX_REMOTE_INSERT_ROWS_PER_STATEMENT, previousRowsPerStatement * 2);
     if (tuning.rowsPerStatement !== previousRowsPerStatement) {
-        Logger.info(`[DataMigrator] Adaptive remote batching: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after ${executedRows} rows in ${formatDuration(durationMs)}`);
+        Logger.info(`[DataMigrator] Remote batch growth: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after ${executedRows} rows in ${formatDuration(durationMs)} with sql_size=${sqlLength}/${tuning.maxStatementSqlLength} bytes`);
     }
 };
-const reduceRemoteBatchTuningAfterFailure = (connection, sqlLength) => {
+const reduceRemoteBatchTuningAfterFailure = (connection) => {
     if (connection.type !== 'd1-remote') {
-        return;
+        return 0;
     }
     const tuning = getRemoteBatchTuning(connection);
     const previousRowsPerStatement = tuning.rowsPerStatement;
-    const nearSqlLimit = sqlLength >= Math.floor(tuning.maxExecutionSqlLength * 0.8);
-    if (!nearSqlLimit && previousRowsPerStatement <= MIN_REMOTE_INSERT_ROWS_PER_STATEMENT) {
-        return;
+    if (previousRowsPerStatement <= MIN_REMOTE_INSERT_ROWS_PER_STATEMENT) {
+        return 0;
     }
-    tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement - 50, Math.floor(previousRowsPerStatement * 0.5)));
+    tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.floor(previousRowsPerStatement / 2));
     if (tuning.rowsPerStatement !== previousRowsPerStatement) {
         Logger.warn(`[DataMigrator] Adaptive remote batching: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after failed remote insert`);
     }
+    return tuning.rowsPerStatement;
 };
 const logTableMigrationProgress = (table, rowsMigrated, totalRows, batchSize) => {
     if (totalRows <= 10000 || rowsMigrated % (batchSize * 10) !== 0) {
@@ -407,18 +438,20 @@ const buildChunkProcessingFailureResult = (sourceConnection, offset, batchSize,
     errors.push(errorMsg);
     if (sourceConnection.driver === 'mysql' && isRetryableConnectionError(error)) {
         const reducedBatchSize = reduceSourceBatchSizeAfterTimeout(sourceConnection, batchSize);
-        return {
-            rowsMigrated: 0,
-            nextOffset: offset,
-            nextBatchSize: reducedBatchSize,
-            continueProcessing: reducedBatchSize >= batchSize,
-        };
+        if (reducedBatchSize > 0) {
+            return {
+                rowsMigrated: 0,
+                nextOffset: offset,
+                nextBatchSize: reducedBatchSize,
+                continueProcessing: true,
+            };
+        }
     }
     return {
         rowsMigrated: 0,
-        nextOffset: offset + batchSize,
+        nextOffset: offset,
         nextBatchSize: batchSize,
-        continueProcessing: true,
+        continueProcessing: false,
     };
 };
 const processTableChunk = async (table, sourceConnection, targetConnection, offset, batchSize, errors) => {
@@ -503,7 +536,7 @@ const executeWithConcurrency = async (items, concurrency, worker) => {
 const estimateRemoteRowSqlLength = (keys, row) => {
     const delimitersLength = keys.length > 0 ? (keys.length - 1) * 2 : 0;
     const valuesLength = keys.reduce((total, key) => {
-        return total + toSqlLiteral(row[key]).length;
+        return total + getSqlByteLength(toSqlLiteral(row[key]));
     }, 0);
     return valuesLength + delimitersLength + 2;
 };
@@ -520,7 +553,7 @@ const createInsertStatements = (targetType, settings, tableName, data) => {
     const statements = [];
     let batchRows = [];
     let batchParameters = [];
-    let batchSqlLength = prefix.length;
+    let batchSqlLength = getSqlByteLength(prefix);
     const flushBatch = () => {
         if (batchRows.length === 0) {
             return;
@@ -529,14 +562,17 @@ const createInsertStatements = (targetType, settings, tableName, data) => {
             sql: `${prefix}${batchRows.map(() => rowPlaceholder).join(', ')}`,
             parameters: batchParameters,
             rowCount: batchRows.length,
+            rows: batchRows,
         });
         batchRows = [];
         batchParameters = [];
-        batchSqlLength = prefix.length;
+        batchSqlLength = getSqlByteLength(prefix);
     };
     for (const row of data) {
         const rowParameters = keys.map((key) => row[key]);
-        const rowSqlLength = targetType === 'd1-remote' ? estimateRemoteRowSqlLength(keys, row) : rowPlaceholder.length;
+        const rowSqlLength = targetType === 'd1-remote'
+            ? estimateRemoteRowSqlLength(keys, row)
+            : getSqlByteLength(rowPlaceholder);
         const separatorLength = batchRows.length > 0 ? 2 : 0;
         const nextSqlLength = batchSqlLength + separatorLength + rowSqlLength;
         if (batchRows.length > 0 && (batchRows.length >= rowLimit || nextSqlLength > maxSqlLength)) {
@@ -549,42 +585,59 @@ const createInsertStatements = (targetType, settings, tableName, data) => {
     flushBatch();
     return statements;
 };
-const createRemoteExecutionBatchesWithLimit = (statements, maxExecutionSqlLength) => {
-    if (statements.length <= 1) {
-        return statements;
-    }
-    const batches = [];
-    let sqlParts = [];
-    let parameters = [];
-    let rowCount = 0;
-    let currentLength = 0;
-    const flushBatch = () => {
-        if (sqlParts.length === 0) {
-            return;
+const executeInsertStatement = async (targetConnection, statement) => {
+    if (!targetConnection.adapter) {
+        throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
+    }
+    const executionStartTime = Date.now();
+    const result = await targetConnection.adapter.query(statement.sql, statement.parameters);
+    const executionDurationMs = Date.now() - executionStartTime;
+    const affectedRows = typeof result.rowCount === 'number' ? result.rowCount : statement.rowCount;
+    return { affectedRows, executionDurationMs };
+};
+const createRemoteInsertExecutionQueue = (targetConnection, tableName, rows) => {
+    const batchSettings = getInsertBatchSettings(targetConnection);
+    return createInsertStatements(targetConnection.type, batchSettings, tableName, rows);
+};
+const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
+    const pendingRows = [data];
+    let insertedRows = 0;
+    while (pendingRows.length > 0) {
+        const currentRows = pendingRows.shift();
+        if (currentRows === undefined || currentRows.length === 0) {
+            continue;
         }
-        batches.push({
-            sql: sqlParts.join(';\n'),
-            parameters,
-            rowCount,
-        });
-        sqlParts = [];
-        parameters = [];
-        rowCount = 0;
-        currentLength = 0;
-    };
-    for (const statement of statements) {
-        const separatorLength = sqlParts.length > 0 ? 2 : 0;
-        const nextLength = currentLength + separatorLength + statement.sql.length;
-        if (sqlParts.length > 0 && nextLength > maxExecutionSqlLength) {
-            flushBatch();
+        const statements = createRemoteInsertExecutionQueue(targetConnection, tableName, currentRows);
+        let shouldRetryCurrentRows = false;
+        for (const [statementIndex, statement] of statements.entries()) {
+            try {
+                const { affectedRows, executionDurationMs } = await executeInsertStatement(targetConnection, statement);
+                insertedRows += affectedRows;
+                adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, getSqlByteLength(statement.sql));
+            }
+            catch (error) {
+                const remainingGroups = statements
+                    .slice(statementIndex + 1)
+                    .map((remainingStatement) => remainingStatement.rows)
+                    .filter((remainingRows) => remainingRows.length > 0);
+                const nextRowsPerStatement = reduceRemoteBatchTuningAfterFailure(targetConnection);
+                if (nextRowsPerStatement <= 0 || statement.rows.length <= 1) {
+                    const innerReason = getErrorMessage(error);
+                    throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}: ${innerReason}`, {
+                        rowCount: statement.rowCount,
+                        reason: innerReason,
+                    });
+                }
+                pendingRows.unshift(statement.rows, ...remainingGroups);
+                shouldRetryCurrentRows = true;
+                break;
+            }
+        }
+        if (shouldRetryCurrentRows) {
+            continue;
         }
-        sqlParts.push(statement.sql);
-        parameters.push(...statement.parameters);
-        rowCount += statement.rowCount;
-        currentLength += (sqlParts.length > 1 ? 2 : 0) + statement.sql.length;
     }
-    flushBatch();
-    return batches;
+    return insertedRows;
 };
 const createRemoteD1Adapter = (database) => {
     return {
@@ -1012,6 +1065,8 @@ export const DataMigrator = Object.freeze({
         try {
             const totalRows = table.rowCount || 0;
             const batchSize = config.batchSize || 1000;
+            resetSourceBatchTuning(sourceConnection, batchSize);
+            resetRemoteBatchTuning(targetConnection);
             const targetRowCount = await DataMigrator.getTargetRowCount(targetConnection, table.name);
             if (targetRowCount >= totalRows) {
                 Logger.info(`Table ${table.name} already synced: ${targetRowCount}/${totalRows} rows, skipping`);
@@ -1024,6 +1079,16 @@ export const DataMigrator = Object.freeze({
                 Logger.info(`Processing ${totalRows} rows in batches of ${batchSize}`);
             }
             rowsMigrated = await DataMigrator.processTableChunks(table, sourceConnection, targetConnection, totalRows, batchSize, targetRowCount, errors);
+            if (errors.length > 0 && rowsMigrated < totalRows) {
+                appendFailedTableReport({
+                    migrationId: config.migrationId || 'unknown',
+                    targetDatabase: targetConnection.database,
+                    tableName: table.name,
+                    rowsMigrated,
+                    totalRows,
+                    errors,
+                });
+            }
             const tableDurationMs = Date.now() - tableStartTime;
             Logger.info(`[DataMigrator] Table ${table.name} completed rows=${rowsMigrated}/${totalRows} duration=${formatDuration(tableDurationMs)} rate=${formatRowsPerSecond(rowsMigrated, tableDurationMs)}`);
             return { rowsMigrated, errors };
@@ -1032,6 +1097,14 @@ export const DataMigrator = Object.freeze({
             const errorMsg = `Table migration failed for ${table.name}: ${error}`;
             Logger.error(errorMsg);
             errors.push(errorMsg);
+            appendFailedTableReport({
+                migrationId: config.migrationId || 'unknown',
+                targetDatabase: targetConnection.database,
+                tableName: table.name,
+                rowsMigrated,
+                totalRows: table.rowCount || 0,
+                errors,
+            });
             return { rowsMigrated, errors };
         }
     },
@@ -1046,8 +1119,7 @@ export const DataMigrator = Object.freeze({
             state.currentBatchSize = getTableChunkBatchSize(sourceConnection, state.retryBatchSize);
             const result = await processTableChunk(table, sourceConnection, targetConnection, state.offset, state.currentBatchSize, errors);
             state = applyTableChunkResult(state, result, batchSize, table, totalRows);
-            if (!result.continueProcessing &&
-                !(result.rowsMigrated === 0 && result.nextBatchSize < batchSize)) {
+            if (!result.continueProcessing) {
                 break;
             }
         }
@@ -1121,33 +1193,17 @@ export const DataMigrator = Object.freeze({
         if (!targetConnection.adapter) {
             throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
         }
+        if (targetConnection.type === 'd1-remote') {
+            return insertRemoteRowsWithRetry(targetConnection, tableName, data);
+        }
         const batchSettings = getInsertBatchSettings(targetConnection);
         const statements = createInsertStatements(targetConnection.type, batchSettings, tableName, data);
-        const executableStatements = targetConnection.type === 'd1-remote'
-            ? createRemoteExecutionBatchesWithLimit(statements, batchSettings.maxExecutionSqlLength)
-            : statements;
+        const executableStatements = statements;
         let insertedRows = 0;
         for (const statement of executableStatements) {
-            try {
-                const executionStartTime = Date.now();
-                const result = await targetConnection.adapter.query(statement.sql, statement.parameters);
-                const executionDurationMs = Date.now() - executionStartTime;
-                const affectedRows = typeof result.rowCount === 'number' ? result.rowCount : statement.rowCount;
-                insertedRows += affectedRows;
-                adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, statement.sql.length);
-            }
-            catch {
-                if (targetConnection.type === 'd1-remote' && data.length > 1 && statement.rowCount > 1) {
-                    reduceRemoteBatchTuningAfterFailure(targetConnection, statement.sql.length);
-                    const midpoint = Math.max(1, Math.floor(data.length / 2));
-                    const left = await DataMigrator.insertData(targetConnection, tableName, data.slice(0, midpoint));
-                    const right = await DataMigrator.insertData(targetConnection, tableName, data.slice(midpoint));
-                    return left + right;
-                }
-                throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
-                    rowCount: statement.rowCount,
-                });
-            }
+            const { affectedRows, executionDurationMs } = await executeInsertStatement(targetConnection, statement);
+            insertedRows += affectedRows;
+            adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, getSqlByteLength(statement.sql));
         }
         return insertedRows;
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zintrust/d1-migrator",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "Resumable database migration toolkit for moving data to Cloudflare D1 with ZinTrust.",
   "private": false,
   "type": "module",