@zintrust/d1-migrator 1.9.2 → 1.9.3

package/dist/cli/DataMigrator.d.ts

@@ -19,11 +19,18 @@ export interface TargetConnection {
     database: string;
     connected: boolean;
     adapter?: DatabaseAdapter;
+    remoteBatchTuning?: RemoteBatchTuning;
 }
 export interface TableInfo {
     name: string;
     rowCount?: number;
+    dependsOn?: string[];
 }
+type RemoteBatchTuning = {
+    rowsPerStatement: number;
+    maxStatementSqlLength: number;
+    maxExecutionSqlLength: number;
+};
 type AdapterQueryResult = {
     rows: Record<string, unknown>[];
     rowCount?: number;
@@ -77,6 +84,7 @@ export declare const DataMigrator: Readonly<{
         rowsMigrated: number;
         errors: string[];
     }>;
+    processTableChunks(table: TableInfo, sourceConnection: SourceConnection, targetConnection: TargetConnection, totalRows: number, batchSize: number, startOffset: number, errors: string[]): Promise<number>;
     /**
      * Read data chunk from source database
      */
@@ -84,7 +92,7 @@ export declare const DataMigrator: Readonly<{
     /**
      * Transform data for D1 compatibility
      */
-    transformData(chunk: Record<string, unknown>[], tableName: string): Promise<Record<string, unknown>[]>;
+    transformData(chunk: Record<string, unknown>[], _tableName: string): Promise<Record<string, unknown>[]>;
     /**
      * Insert data into target database
      */

package/dist/cli/DataMigrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AA8XF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAsFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA+BzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAuChB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAmBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IA6FtD;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAkBrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,aACrB,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA4CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAkClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
+{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAQD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAssBF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAoGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4BzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAuChB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BAoD7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA6DlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAgBrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAiDlB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}

package/dist/cli/DataMigrator.js

@@ -22,6 +22,35 @@ const extractWranglerJson = (output) => {
         return null;
     }
 };
+const normalizeWranglerTableValue = (value) => {
+    const trimmed = value.trim();
+    if (trimmed === '') {
+        return '';
+    }
+    const normalized = trimmed.toLowerCase();
+    if (normalized === 'null') {
+        return null;
+    }
+    if (normalized === 'true') {
+        return true;
+    }
+    if (normalized === 'false') {
+        return false;
+    }
+    if (/^-?\d+$/.test(trimmed)) {
+        const parsed = Number.parseInt(trimmed, 10);
+        if (Number.isSafeInteger(parsed)) {
+            return parsed;
+        }
+    }
+    if (/^-?(?:\d+\.\d+|\d+\.\d*|\.\d+)$/.test(trimmed)) {
+        const parsed = Number.parseFloat(trimmed);
+        if (Number.isFinite(parsed)) {
+            return parsed;
+        }
+    }
+    return trimmed;
+};
 const parseWranglerTable = (output) => {
     const lines = output.split('\n').map((line) => line.trim());
     const dataLines = lines.filter((line) => line.startsWith('│') && line.endsWith('│'));
@@ -49,6 +78,15 @@ const parseWranglerTable = (output) => {
     }
     return rows;
 };
+const parseWranglerTableRows = (output) => {
+    return parseWranglerTable(output).map((row) => {
+        const normalizedRow = {};
+        for (const [key, value] of Object.entries(row)) {
+            normalizedRow[key] = normalizeWranglerTableValue(value);
+        }
+        return normalizedRow;
+    });
+};
 const toHex = (value) => {
     return Array.from(value)
         .map((byte) => byte.toString(16).padStart(2, '0'))
@@ -94,6 +132,228 @@ const bindSqlParameters = (sql, parameters) => {
         return rendered;
     });
 };
+const REMOTE_INSERT_ROWS_PER_STATEMENT = 200;
+const LOCAL_INSERT_ROWS_PER_STATEMENT = 500;
+const MAX_REMOTE_INSERT_SQL_LENGTH = 100000;
+const MAX_REMOTE_EXECUTION_SQL_LENGTH = 250000;
+const MIN_REMOTE_INSERT_ROWS_PER_STATEMENT = 50;
+const MAX_REMOTE_INSERT_ROWS_PER_STATEMENT = 2000;
+const DEFAULT_REMOTE_TABLE_PARALLELISM = 4;
+const formatDuration = (durationMs) => {
+    if (durationMs < 1000) {
+        return `${durationMs}ms`;
+    }
+    return `${(durationMs / 1000).toFixed(durationMs < 10000 ? 2 : 1)}s`;
+};
+const formatRowsPerSecond = (rows, durationMs) => {
+    if (rows <= 0 || durationMs <= 0) {
+        return 'n/a';
+    }
+    const rate = rows / (durationMs / 1000);
+    return `${rate >= 100 ? rate.toFixed(0) : rate.toFixed(2)} rows/s`;
+};
+const getRemoteBatchTuning = (connection) => {
+    if (connection.remoteBatchTuning !== undefined) {
+        return connection.remoteBatchTuning;
+    }
+    connection.remoteBatchTuning = {
+        rowsPerStatement: REMOTE_INSERT_ROWS_PER_STATEMENT,
+        maxStatementSqlLength: MAX_REMOTE_INSERT_SQL_LENGTH,
+        maxExecutionSqlLength: MAX_REMOTE_EXECUTION_SQL_LENGTH,
+    };
+    return connection.remoteBatchTuning;
+};
+const getInsertBatchSettings = (connection) => {
+    if (connection.type === 'd1-remote') {
+        const tuning = getRemoteBatchTuning(connection);
+        return {
+            rowsPerStatement: tuning.rowsPerStatement,
+            maxStatementSqlLength: tuning.maxStatementSqlLength,
+            maxExecutionSqlLength: tuning.maxExecutionSqlLength,
+        };
+    }
+    return {
+        rowsPerStatement: LOCAL_INSERT_ROWS_PER_STATEMENT,
+        maxStatementSqlLength: Number.POSITIVE_INFINITY,
+        maxExecutionSqlLength: Number.POSITIVE_INFINITY,
+    };
+};
+const adjustRemoteBatchTuning = (connection, executedRows, durationMs, sqlLength) => {
+    if (connection.type !== 'd1-remote' || executedRows <= 0) {
+        return;
+    }
+    const tuning = getRemoteBatchTuning(connection);
+    const previousRowsPerStatement = tuning.rowsPerStatement;
+    const nearSqlLimit = sqlLength >= Math.floor(tuning.maxExecutionSqlLength * 0.9);
+    if (durationMs <= 1500 &&
+        executedRows >= Math.floor(previousRowsPerStatement * 0.8) &&
+        !nearSqlLimit) {
+        tuning.rowsPerStatement = Math.min(MAX_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.max(previousRowsPerStatement + 25, Math.floor(previousRowsPerStatement * 1.25)));
+    }
+    else if (durationMs >= 6000 || nearSqlLimit) {
+        tuning.rowsPerStatement = Math.max(MIN_REMOTE_INSERT_ROWS_PER_STATEMENT, Math.min(previousRowsPerStatement - 25, Math.floor(previousRowsPerStatement * 0.75)));
+    }
+    if (tuning.rowsPerStatement !== previousRowsPerStatement) {
+        Logger.info(`[DataMigrator] Adaptive remote batching: rows_per_statement ${previousRowsPerStatement} -> ${tuning.rowsPerStatement} after ${executedRows} rows in ${formatDuration(durationMs)}`);
+    }
+};
+const getTableDependencies = (table) => {
+    if (!Array.isArray(table.dependsOn)) {
+        return [];
+    }
+    return [...new Set(table.dependsOn.filter((dependency) => dependency.trim() !== ''))];
+};
+const buildTableMigrationLevels = (tables) => {
+    const tablesByName = new Map();
+    for (const table of tables) {
+        tablesByName.set(table.name, table);
+    }
+    const unresolved = new Set(tables.map((table) => table.name));
+    const dependenciesByTable = new Map();
+    for (const table of tables) {
+        dependenciesByTable.set(table.name, new Set(getTableDependencies(table).filter((dependency) => dependency !== table.name && tablesByName.has(dependency))));
+    }
+    const levels = [];
+    while (unresolved.size > 0) {
+        const readyNames = tables
+            .map((table) => table.name)
+            .filter((name) => unresolved.has(name) && (dependenciesByTable.get(name)?.size ?? 0) === 0);
+        if (readyNames.length === 0) {
+            const cyclicTables = tables.filter((table) => unresolved.has(table.name));
+            if (cyclicTables.length > 0) {
+                Logger.warn(`[DataMigrator] Table dependency cycle or unresolved reference detected. Falling back to sequential execution for: ${cyclicTables.map((table) => table.name).join(', ')}`);
+                levels.push(...cyclicTables.map((table) => [table]));
+            }
+            break;
+        }
+        const levelTables = readyNames
+            .map((name) => tablesByName.get(name))
+            .filter((table) => table !== undefined);
+        levels.push(levelTables);
+        for (const readyName of readyNames) {
+            unresolved.delete(readyName);
+        }
+        for (const name of unresolved) {
+            const dependencies = dependenciesByTable.get(name);
+            for (const readyName of readyNames) {
+                dependencies?.delete(readyName);
+            }
+        }
+    }
+    return levels;
+};
+const getTableParallelism = (config, targetConnection) => {
+    if (targetConnection.type !== 'd1-remote') {
+        return 1;
+    }
+    if (config.sourceDriver === 'sqlite') {
+        return 1;
+    }
+    return DEFAULT_REMOTE_TABLE_PARALLELISM;
+};
+const executeWithConcurrency = async (items, concurrency, worker) => {
+    if (items.length === 0) {
+        return [];
+    }
+    const effectiveConcurrency = Math.max(1, Math.min(concurrency, items.length));
+    const results = new Array(items.length);
+    let index = 0;
+    const runWorker = async () => {
+        while (index < items.length) {
+            const currentIndex = index;
+            index += 1;
+            results[currentIndex] = await worker(items[currentIndex]);
+        }
+    };
+    await Promise.all(Array.from({ length: effectiveConcurrency }, () => runWorker()));
+    return results;
+};
+const estimateRemoteRowSqlLength = (keys, row) => {
+    const delimitersLength = keys.length > 0 ? (keys.length - 1) * 2 : 0;
+    const valuesLength = keys.reduce((total, key) => {
+        return total + toSqlLiteral(row[key]).length;
+    }, 0);
+    return valuesLength + delimitersLength + 2;
+};
+const createInsertStatements = (targetType, settings, tableName, data) => {
+    if (data.length === 0) {
+        return [];
+    }
+    const keys = Object.keys(data[0]);
+    const columnList = keys.map((key) => `\`${key}\``).join(', ');
+    const rowPlaceholder = `(${keys.map(() => '?').join(', ')})`;
+    const prefix = `INSERT INTO \`${tableName}\` (${columnList}) VALUES `;
+    const rowLimit = settings.rowsPerStatement;
+    const maxSqlLength = settings.maxStatementSqlLength;
+    const statements = [];
+    let batchRows = [];
+    let batchParameters = [];
+    let batchSqlLength = prefix.length;
+    const flushBatch = () => {
+        if (batchRows.length === 0) {
+            return;
+        }
+        statements.push({
+            sql: `${prefix}${batchRows.map(() => rowPlaceholder).join(', ')}`,
+            parameters: batchParameters,
+            rowCount: batchRows.length,
+        });
+        batchRows = [];
+        batchParameters = [];
+        batchSqlLength = prefix.length;
+    };
+    for (const row of data) {
+        const rowParameters = keys.map((key) => row[key]);
+        const rowSqlLength = targetType === 'd1-remote' ? estimateRemoteRowSqlLength(keys, row) : rowPlaceholder.length;
+        const separatorLength = batchRows.length > 0 ? 2 : 0;
+        const nextSqlLength = batchSqlLength + separatorLength + rowSqlLength;
+        if (batchRows.length > 0 && (batchRows.length >= rowLimit || nextSqlLength > maxSqlLength)) {
+            flushBatch();
+        }
+        batchRows.push(row);
+        batchParameters.push(...rowParameters);
+        batchSqlLength += (batchRows.length > 1 ? 2 : 0) + rowSqlLength;
+    }
+    flushBatch();
+    return statements;
+};
+const createRemoteExecutionBatchesWithLimit = (statements, maxExecutionSqlLength) => {
+    if (statements.length <= 1) {
+        return statements;
+    }
+    const batches = [];
+    let sqlParts = [];
+    let parameters = [];
+    let rowCount = 0;
+    let currentLength = 0;
+    const flushBatch = () => {
+        if (sqlParts.length === 0) {
+            return;
+        }
+        batches.push({
+            sql: sqlParts.join(';\n'),
+            parameters,
+            rowCount,
+        });
+        sqlParts = [];
+        parameters = [];
+        rowCount = 0;
+        currentLength = 0;
+    };
+    for (const statement of statements) {
+        const separatorLength = sqlParts.length > 0 ? 2 : 0;
+        const nextLength = currentLength + separatorLength + statement.sql.length;
+        if (sqlParts.length > 0 && nextLength > maxExecutionSqlLength) {
+            flushBatch();
+        }
+        sqlParts.push(statement.sql);
+        parameters.push(...statement.parameters);
+        rowCount += statement.rowCount;
+        currentLength += (sqlParts.length > 1 ? 2 : 0) + statement.sql.length;
+    }
+    flushBatch();
+    return batches;
+};
 const createRemoteD1Adapter = (database) => {
     return {
         async connect() {
@@ -107,38 +367,22 @@ const createRemoteD1Adapter = (database) => {
             const output = WranglerD1.executeSql({ dbName: database, isLocal: false, sql: renderedSql });
             const payload = extractWranglerJson(output);
             if (payload === null || payload.length === 0) {
-                const rows = parseWranglerTable(output).map((row) => row);
+                const rows = parseWranglerTableRows(output);
                 return { rows, rowCount: rows.length };
             }
-            const last = payload[payload.length - 1];
+            const last = payload.at(-1);
+            if (last === undefined) {
+                return { rows: [], rowCount: 0 };
+            }
             const rows = Array.isArray(last.results) ? last.results : [];
-            const changes = last.meta?.changes;
-            const rowCount = typeof changes === 'number' ? changes : rows.length;
+            const totalChanges = payload.reduce((count, statement) => {
+                return count + (typeof statement.meta?.changes === 'number' ? statement.meta.changes : 0);
+            }, 0);
+            const rowCount = totalChanges > 0 ? totalChanges : rows.length;
             return { rows, rowCount };
         },
     };
 };
-const redactConnectionString = (connectionString) => {
-    try {
-        const parsed = new URL(connectionString);
-        if (parsed.password.trim() !== '') {
-            parsed.password = '***';
-        }
-        return parsed.toString();
-    }
-    catch {
-        return connectionString;
-    }
-};
-const getUrlPasswordForm = (connectionString) => {
-    try {
-        const parsed = new URL(connectionString);
-        return parsed.password || '';
-    }
-    catch {
-        return undefined;
-    }
-};
 const getErrorCause = (error) => {
     if (error === null || typeof error !== 'object') {
         return undefined;
@@ -190,11 +434,6 @@ const logDetailedError = (label, error) => {
         logDetailedError(`${label} cause`, cause);
     }
 };
-const describePasswordForLog = (password) => {
-    const hasSpecialCharacters = /[^a-zA-Z0-9]/.test(password);
-    const containsBang = password.includes('!');
-    return `len=${password.length}, special_chars=${hasSpecialCharacters}, contains_bang=${containsBang}`;
-};
 const normalizeNullLikeValue = (value) => {
     if (typeof value !== 'string')
         return value;
@@ -240,11 +479,6 @@ const createSourceAdapter = (config) => {
     switch (config.sourceDriver) {
         case 'mysql': {
             const connectionDetails = parseConnectionDetails(config.sourceConnection, 3306, 'mysql', 'root');
-            const urlPasswordForm = getUrlPasswordForm(config.sourceConnection);
-            Logger.info(`[DataMigrator] Source password diagnostics: ${describePasswordForLog(connectionDetails.password)}`);
-            if (urlPasswordForm !== undefined) {
-                Logger.info(`[DataMigrator] MySQL password handoff: url_form(${describePasswordForLog(urlPasswordForm)}), final_auth(${describePasswordForLog(connectionDetails.password)}), matches=${urlPasswordForm === connectionDetails.password}`);
-            }
             return MySQLAdapter.create({
                 driver: 'mysql',
                 host: connectionDetails.host,
@@ -335,14 +569,20 @@ export const DataMigrator = Object.freeze({
             if (targetConnection.adapter) {
                 await DataMigrator.prepareTargetSchema(sourceConnection, targetConnection, config);
             }
-            // Migrate each table sequentially for reliable D1/SQLite writes
             Logger.info('Starting table migration...');
-            for (const table of schema.tables) {
-                const result = await DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
-                progress.processedRows += result.rowsMigrated;
-                // Add any errors to progress
-                if (result.errors.length > 0) {
-                    progress.errors[table.name] = result.errors.join('; ');
+            const tableLevels = buildTableMigrationLevels(schema.tables);
+            const tableParallelism = getTableParallelism(config, targetConnection);
+            for (const [levelIndex, tables] of tableLevels.entries()) {
+                Logger.info(`[DataMigrator] Starting table level ${levelIndex + 1}/${tableLevels.length}: ${tables.map((table) => table.name).join(', ')}`);
+                const levelResults = await executeWithConcurrency(tables, tableParallelism, async (table) => {
+                    return DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
+                });
+                for (const [resultIndex, result] of levelResults.entries()) {
+                    const table = tables[resultIndex];
+                    progress.processedRows += result.rowsMigrated;
+                    if (result.errors.length > 0 && table !== undefined) {
+                        progress.errors[table.name] = result.errors.join('; ');
+                    }
                 }
             }
             progress.totalRows = Math.max(progress.totalRows, progress.processedRows);
@@ -372,7 +612,6 @@ export const DataMigrator = Object.freeze({
      */
     async connectToSource(config) {
         Logger.info(`Connecting to ${config.sourceDriver} database...`);
-        Logger.info(`[DataMigrator] Source connection (redacted): ${redactConnectionString(config.sourceConnection)}`);
         const adapter = createSourceAdapter(config);
         try {
             await adapter.connect();
@@ -473,6 +712,7 @@ export const DataMigrator = Object.freeze({
         const tables = sourceSchema.tables.map((table) => ({
             name: table.name,
             rowCount: table.rowCount || 0,
+            dependsOn: table.foreignKeys.map((foreignKey) => foreignKey.referencedTable),
         }));
         Logger.info(`Found ${tables.length} tables`);
         return { tables };
@@ -500,10 +740,10 @@ export const DataMigrator = Object.freeze({
         Logger.info(`Migrating table: ${table.name}`);
         const errors = [];
         let rowsMigrated = 0;
+        const tableStartTime = Date.now();
         try {
             const totalRows = table.rowCount || 0;
             const batchSize = config.batchSize || 1000;
-            // Check if table is already synced for resumability
             const targetRowCount = await DataMigrator.getTargetRowCount(targetConnection, table.name);
             if (targetRowCount >= totalRows) {
                 Logger.info(`Table ${table.name} already synced: ${targetRowCount}/${totalRows} rows, skipping`);
@@ -515,52 +755,54 @@ export const DataMigrator = Object.freeze({
             else {
                 Logger.info(`Processing ${totalRows} rows in batches of ${batchSize}`);
             }
-            // Process data in chunks sequentially for data integrity
-            // Start from the last synced offset for resumability
-            const startOffset = targetRowCount;
-            for (let offset = startOffset; offset < totalRows; offset += batchSize) {
-                try {
-                    const chunk = await DataMigrator.readDataChunk(sourceConnection, table.name, offset, batchSize);
-                    if (chunk.length === 0)
-                        break;
-                    // Transform data for D1 compatibility
-                    const transformedChunk = await DataMigrator.transformData(chunk, table.name);
-                    // Insert data into target
-                    const insertedRows = await DataMigrator.insertData(targetConnection, table.name, transformedChunk);
-                    if (insertedRows !== chunk.length) {
-                        const verificationError = DataMigrator.createChunkVerificationError(table.name, offset, chunk.length, insertedRows);
-                        throw ErrorFactory.createValidationError(`Chunk insert mismatch on ${table.name}`, verificationError);
-                    }
-                    rowsMigrated += insertedRows;
-                    // Log progress for large tables
-                    if (totalRows > 10000 && rowsMigrated % (batchSize * 10) === 0) {
-                        const normalizedTotalRows = Math.max(totalRows, rowsMigrated);
-                        const percentage = Math.round((rowsMigrated / normalizedTotalRows) * 100);
-                        Logger.info(`Table ${table.name}: ${rowsMigrated}/${normalizedTotalRows} (${percentage}%)`);
-                    }
-                }
-                catch (error) {
-                    const errorMsg = `Chunk processing failed at offset ${offset}: ${error}`;
-                    Logger.error(errorMsg);
-                    errors.push(errorMsg);
-                    // Continue with next chunk instead of failing completely
-                    continue;
-                }
-            }
-            Logger.info(`Table ${table.name} completed: ${rowsMigrated} rows migrated`);
+            rowsMigrated = await DataMigrator.processTableChunks(table, sourceConnection, targetConnection, totalRows, batchSize, targetRowCount, errors);
+            const tableDurationMs = Date.now() - tableStartTime;
+            Logger.info(`[DataMigrator] Table ${table.name} completed rows=${rowsMigrated}/${totalRows} duration=${formatDuration(tableDurationMs)} rate=${formatRowsPerSecond(rowsMigrated, tableDurationMs)}`);
+            return { rowsMigrated, errors };
         }
         catch (error) {
-            const errorMsg = `Failed to migrate table ${table.name}: ${error}`;
+            const errorMsg = `Table migration failed for ${table.name}: ${error}`;
             Logger.error(errorMsg);
             errors.push(errorMsg);
+            return { rowsMigrated, errors };
+        }
+    },
+    async processTableChunks(table, sourceConnection, targetConnection, totalRows, batchSize, startOffset, errors) {
+        let rowsMigrated = 0;
+        for (let offset = startOffset; offset < totalRows; offset += batchSize) {
+            try {
+                const chunkStartTime = Date.now();
+                const chunk = await DataMigrator.readDataChunk(sourceConnection, table.name, offset, batchSize);
+                if (chunk.length === 0) {
+                    break;
+                }
+                const transformedChunk = await DataMigrator.transformData(chunk, table.name);
+                const insertedRows = await DataMigrator.insertData(targetConnection, table.name, transformedChunk);
+                if (insertedRows !== chunk.length) {
+                    const verificationError = DataMigrator.createChunkVerificationError(table.name, offset, chunk.length, insertedRows);
+                    throw ErrorFactory.createValidationError(`Chunk insert mismatch on ${table.name}`, verificationError);
+                }
+                rowsMigrated += insertedRows;
+                const chunkDurationMs = Date.now() - chunkStartTime;
+                Logger.info(`[DataMigrator] Chunk ${table.name} offset=${offset} rows=${insertedRows} duration=${formatDuration(chunkDurationMs)} rate=${formatRowsPerSecond(insertedRows, chunkDurationMs)}`);
+                if (totalRows > 10000 && rowsMigrated % (batchSize * 10) === 0) {
+                    const normalizedTotalRows = Math.max(totalRows, rowsMigrated);
+                    const percentage = Math.round((rowsMigrated / normalizedTotalRows) * 100);
+                    Logger.info(`Table ${table.name}: ${rowsMigrated}/${normalizedTotalRows} (${percentage}%)`);
+                }
+            }
+            catch (error) {
+                const errorMsg = `Chunk processing failed at offset ${offset}: ${error}`;
+                Logger.error(errorMsg);
+                errors.push(errorMsg);
+            }
         }
-        return { rowsMigrated, errors };
+        return rowsMigrated;
     },
     /**
      * Read data chunk from source database
      */
     async readDataChunk(sourceConnection, tableName, offset, batchSize) {
-        Logger.debug(`Reading chunk from ${tableName}: offset ${offset}, size ${batchSize}`);
         if (!sourceConnection.adapter)
             return [];
         try {
@@ -576,8 +818,7 @@ export const DataMigrator = Object.freeze({
     /**
      * Transform data for D1 compatibility
      */
-    async transformData(chunk, tableName) {
-        Logger.debug(`Transforming ${chunk.length} rows for table ${tableName}`);
+    async transformData(chunk, _tableName) {
         return chunk.map((row) => {
             const transformed = {};
             for (const [key, rawValue] of Object.entries(row)) {
@@ -612,27 +853,30 @@ export const DataMigrator = Object.freeze({
      * Insert data into target database
      */
     async insertData(targetConnection, tableName, data) {
-        Logger.debug(`Inserting ${data.length} rows into ${tableName}`);
         if (data.length === 0)
             return 0;
         if (!targetConnection.adapter) {
             throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
         }
-        const keys = Object.keys(data[0]);
-        const columnList = keys.map((key) => `\`${key}\``).join(', ');
-        const placeholders = keys.map(() => '?').join(', ');
-        const sql = `INSERT INTO \`${tableName}\` (${columnList}) VALUES (${placeholders})`;
+        const batchSettings = getInsertBatchSettings(targetConnection);
+        const statements = createInsertStatements(targetConnection.type, batchSettings, tableName, data);
+        const executableStatements = targetConnection.type === 'd1-remote'
+            ? createRemoteExecutionBatchesWithLimit(statements, batchSettings.maxExecutionSqlLength)
+            : statements;
         let insertedRows = 0;
-        for (const row of data) {
-            const values = keys.map((key) => row[key]);
+        for (const statement of executableStatements) {
             try {
-                await targetConnection.adapter.query(sql, values);
-                insertedRows += 1;
+                const executionStartTime = Date.now();
+                const result = await targetConnection.adapter.query(statement.sql, statement.parameters);
+                const executionDurationMs = Date.now() - executionStartTime;
+                const affectedRows = typeof result.rowCount === 'number' ? result.rowCount : statement.rowCount;
+                insertedRows += affectedRows;
+                adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, statement.sql.length);
             }
             catch (error) {
                 throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
-                    sql,
-                    row,
+                    sql: statement.sql,
+                    rowCount: statement.rowCount,
                     cause: error,
                 });
             }

package/dist/cli/MigrateToD1Command.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,UAAU,CAAC;AAOxE,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAoyBF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,iBAiH/B,CAAC;AAEH;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAetE;AAED;;GAEG;AACH,iBAAe,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAmGxE;AAED;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGtE;AAED;;GAEG;AACH,iBAAS,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrF;AAGD,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC"}
+{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,UAAU,CAAC;AAOxE,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAouBF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,iBA0G/B,CAAC;AAEH;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAetE;AAED;;GAEG;AACH,iBAAe,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAmGxE;AAED;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGtE;AAED;;GAEG;AACH,iBAAS,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrF;AAGD,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC"}

package/dist/cli/MigrateToD1Command.js

@@ -229,35 +229,6 @@ const normalizeSourceConnectionString = (sourceConnection, sourceDriver, origin)
     }
     return buildNetworkConnectionString(details);
 };
-const redactConnectionString = (sourceConnection) => {
-    try {
-        const parsed = new URL(sourceConnection);
-        if (parsed.password.trim() !== '') {
-            parsed.password = '***';
-        }
-        return parsed.toString();
-    }
-    catch {
-        return sourceConnection;
-    }
-};
-const getPasswordForDiagnostics = (sourceConnection, sourceDriver, origin) => {
-    if (origin === 'db-env') {
-        return parseNetworkConnectionDetails(sourceConnection, sourceDriver)?.password;
-    }
-    try {
-        const parsed = new URL(sourceConnection);
-        return parsed.password;
-    }
-    catch {
-        return undefined;
-    }
-};
-const describePasswordForLog = (password) => {
-    const hasSpecialCharacters = /[^a-zA-Z0-9]/.test(password);
-    const containsBang = password.includes('!');
-    return `len=${password.length}, special_chars=${hasSpecialCharacters}, contains_bang=${containsBang}`;
-};
 const getErrorCause = (error) => {
     if (error === null || typeof error !== 'object') {
         return undefined;
@@ -309,18 +280,6 @@ const logDetailedError = (label, error) => {
         logDetailedError(`${label} cause`, cause);
     }
 };
-const logSourceConnectionDiagnostics = (sourceDriver, sourceConnection, origin, originalValue) => {
-    Logger.info(`[d1-migrator] Source connection origin: ${origin}`);
-    Logger.info(`[d1-migrator] Source connection driver: ${sourceDriver}`);
-    Logger.info(`[d1-migrator] Source connection (redacted): ${redactConnectionString(sourceConnection)}`);
-    const originalPassword = getPasswordForDiagnostics(originalValue, sourceDriver, origin);
-    const finalPassword = getPasswordForDiagnostics(sourceConnection, sourceDriver, origin);
-    if (originalPassword === undefined || finalPassword === undefined) {
-        Logger.info('[d1-migrator] Source connection diagnostics: non-network source or unable to parse URL');
-        return;
-    }
-    Logger.info(`[d1-migrator] Source password diagnostics: provided(${describePasswordForLog(originalPassword)}), final(${describePasswordForLog(finalPassword)}), matches=${originalPassword === finalPassword}`);
-};
 const normalizeSourceDriver = (value) => {
     if (value === undefined) {
         return undefined;
@@ -578,8 +537,7 @@ export const MigrateToD1Command = BaseCommand.create({
     execute: async (options) => {
         try {
             Logger.info('Starting D1 migration process...');
-            const { config, schemaOnly, sourceConnectionOrigin, originalSourceConnection } = resolveMigrationConfig(options);
-            logSourceConnectionDiagnostics(config.sourceDriver, config.sourceConnection, sourceConnectionOrigin, originalSourceConnection);
+            const { config, schemaOnly } = resolveMigrationConfig(options);
             const configValidation = validateConfig(config);
             if (!configValidation.valid) {
                 throw ErrorFactory.createValidationError(`Invalid migration configuration: ${configValidation.errors.join(', ')}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zintrust/d1-migrator",
-  "version": "1.9.2",
+  "version": "1.9.3",
   "description": "Resumable database migration toolkit for moving data to Cloudflare D1 with ZinTrust.",
   "private": false,
   "type": "module",
@@ -41,7 +41,7 @@
   },
   "dependencies": {
     "@zintrust/db-d1": "^1.8.0",
-    "@zintrust/db-mysql": "^1.8.9",
+    "@zintrust/db-mysql": "^1.9.0",
     "@zintrust/db-postgres": "^1.8.0",
     "@zintrust/db-sqlite": "^1.8.0",
     "@zintrust/db-sqlserver": "^1.8.0"