@zintrust/d1-migrator 2.0.3 → 2.0.4

package/README.md

@@ -226,6 +226,9 @@ The command supports env-based execution for all CLI settings.
 | Interactive (`--interactive`)                 | `MIGRATE_TO_D1_INTERACTIVE`, `D1_MIGRATOR_INTERACTIVE`                                                                         |
 | Resume (`--resume`)                           | `MIGRATE_TO_D1_RESUME`, `D1_MIGRATOR_RESUME`                                                                                   |
 | Migration ID (`--migration-id`)               | `MIGRATE_TO_D1_MIGRATION_ID`, `D1_MIGRATOR_MIGRATION_ID`                                                                       |
+| Group Small Tables                            | `MIGRATE_TO_D1_GROUP_SMALL_TABLES` (Default: `true`)                                                                           |
+| Max Group Rows                                | `MIGRATE_TO_D1_MAX_GROUP_ROWS` (Default: `50000`)                                                                              |
+| Max Group Size (MB)                           | `MIGRATE_TO_D1_MAX_GROUP_SIZE_MB` (Default: `10`)                                                                              |
 
 If `--source-connection` is not provided, the command automatically composes a URI from `DB_*` values for MySQL/PostgreSQL/SQL Server, and uses `DB_PATH`/`DB_DATABASE` for SQLite. Host fallback prefers `DB_READ_HOSTS`, then `DB_HOSTS`, then `DB_HOST`.
 

package/dist/cli/DataMigrator.d.ts

@@ -30,6 +30,7 @@ export interface TargetConnection {
     connected: boolean;
     adapter?: DatabaseAdapter;
     remoteBatchTuning?: RemoteBatchTuning;
+    targetRowCountsCache?: Record<string, number>;
 }
 export interface TableInfo {
     name: string;
@@ -83,6 +84,10 @@ export declare const DataMigrator: Readonly<{
     getSchemaInfo(_connection: SourceConnection): Promise<{
         tables: TableInfo[];
     }>;
+    /**
+     * Populate target table row counts cache to optimize performance and prevent countless individual DB count queries
+     */
+    populateTargetRowCountsCache(targetConnection: TargetConnection, tables: TableInfo[]): Promise<void>;
     /**
      * Get target table row count for resumability
      */

package/dist/cli/DataMigrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAysCF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAmGtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IA+EhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BA0E7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA8BlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAyClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
+{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAglDF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAqFhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;mDAEiB,gBAAgB,UAC1B,SAAS,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAgChB;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BA0E7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA8BlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAuDlB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}

package/dist/cli/DataMigrator.js

@@ -4,7 +4,7 @@
  * Handles the actual data migration between databases
  */
 import { ErrorFactory, LocalD1Resolver, Logger, WranglerD1 } from '@zintrust/core';
-import { appendFileSync, mkdirSync } from 'node:fs';
+import { appendFileSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs';
 import * as path from 'node:path';
 import { MySQLAdapter } from '@zintrust/db-mysql';
 import { PostgreSQLAdapter } from '@zintrust/db-postgres';
@@ -519,6 +519,153 @@ const buildTableMigrationLevels = (tables) => {
     }
     return levels;
 };
+function groupTablesByLimits(tables) {
+    const isGroupingEnabled = (process.env['MIGRATE_TO_D1_GROUP_SMALL_TABLES'] ?? 'true') === 'true';
+    if (!isGroupingEnabled) {
+        return { largeTables: tables, groups: [] };
+    }
+    const maxGroupRows = Number.parseInt(process.env['MIGRATE_TO_D1_MAX_GROUP_ROWS'] ?? '50000', 10);
+    const maxGroupSizeBytes = Number.parseInt(process.env['MIGRATE_TO_D1_MAX_GROUP_SIZE_MB'] ?? '10', 10) * 1024 * 1024;
+    const largeTables = [];
+    const groups = [];
+    let currentGroup = [];
+    let currentGroupRows = 0;
+    let currentGroupBytes = 0;
+    for (const table of tables) {
+        const rowCount = table.rowCount || 0;
+        const estimatedBytes = rowCount * 400;
+        if (rowCount > maxGroupRows || estimatedBytes > maxGroupSizeBytes) {
+            largeTables.push(table);
+            continue;
+        }
+        const exceedsRows = currentGroupRows + rowCount > maxGroupRows;
+        const exceedsBytes = currentGroupBytes + estimatedBytes > maxGroupSizeBytes;
+        if (exceedsRows || exceedsBytes) {
+            if (currentGroup.length > 0) {
+                groups.push(currentGroup);
+            }
+            currentGroup = [table];
+            currentGroupRows = rowCount;
+            currentGroupBytes = estimatedBytes;
+        }
+        else {
+            currentGroup.push(table);
+            currentGroupRows += rowCount;
+            currentGroupBytes += estimatedBytes;
+        }
+    }
+    if (currentGroup.length > 0) {
+        groups.push(currentGroup);
+    }
+    const finalGroups = groups.filter((group) => {
+        if (group.length === 1 && group[0] !== undefined) {
+            largeTables.push(group[0]);
+            return false;
+        }
+        return true;
+    });
+    return { largeTables, groups: finalGroups };
+}
+const processSmallTableRows = async (table, sourceConnection, targetConnection, rowsMigratedMap, allSqlStatements) => {
+    const totalRows = table.rowCount || 0;
+    const targetRowCount = await DataMigrator.getTargetRowCount(targetConnection, table.name);
+    if (targetRowCount >= totalRows) {
+        Logger.info(`[DataMigrator] Table ${table.name} already synced: ${targetRowCount}/${totalRows} rows, skipping in group`);
+        rowsMigratedMap[table.name] = totalRows;
+        return;
+    }
+    const rows = await DataMigrator.readDataChunk(sourceConnection, table.name, 0, totalRows);
+    if (rows.length === 0) {
+        rowsMigratedMap[table.name] = 0;
+        return;
+    }
+    const transformedRows = await DataMigrator.transformData(rows, table.name);
+    const batchSettings = getInsertBatchSettings(targetConnection);
+    const statements = createInsertStatements(targetConnection.type, batchSettings, table.name, transformedRows);
+    for (const statement of statements) {
+        allSqlStatements.push(bindSqlParameters(statement.sql, statement.parameters));
+    }
+    rowsMigratedMap[table.name] = rows.length;
+};
+async function migrateSmallTablesGroup(tables, sourceConnection, targetConnection, config) {
+    Logger.info(`[DataMigrator] Group migrating ${tables.length} small tables: ${tables.map((t) => t.name).join(', ')}`);
+    const errors = [];
+    const rowsMigratedMap = {};
+    const tempFileDir = path.join('.wrangler', 'tmp');
+    const tempFileName = `migration-group-${Date.now()}-${Math.random().toString(36).substring(7)}.sql`;
+    const tempFilePath = path.join(tempFileDir, tempFileName);
+    try {
+        mkdirSync(tempFileDir, { recursive: true });
+        const allSqlStatements = [];
+        for (const table of tables) {
+            await processSmallTableRows(table, sourceConnection, targetConnection, rowsMigratedMap, allSqlStatements);
+        }
+        if (allSqlStatements.length > 0) {
+            const sqlContent = allSqlStatements.join(';\n') + ';';
+            writeFileSync(tempFilePath, sqlContent, 'utf8');
+            const executionStartTime = Date.now();
+            WranglerD1.executeSql({
+                dbName: targetConnection.database,
+                isLocal: targetConnection.type !== 'd1-remote',
+                file: tempFilePath,
+            });
+            const durationMs = Date.now() - executionStartTime;
+            if (targetConnection.targetRowCountsCache) {
+                for (const table of tables) {
+                    targetConnection.targetRowCountsCache[table.name] = table.rowCount || 0;
+                }
+            }
+            Logger.info(`[DataMigrator] Successfully migrated group of ${tables.length} tables in ${formatDuration(durationMs)}`);
+        }
+        try {
+            unlinkSync(tempFilePath);
+        }
+        catch {
+            // Ignore
+        }
+        return { rowsMigratedMap, errors };
+    }
+    catch (error) {
+        try {
+            unlinkSync(tempFilePath);
+        }
+        catch {
+            // Ignore
+        }
+        const errMsg = `Group migration failed: ${getErrorMessage(error)}`;
+        Logger.warn(`[DataMigrator] ${errMsg}. Falling back to individual table migration...`);
+        for (const table of tables) {
+            const result = await DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
+            rowsMigratedMap[table.name] = result.rowsMigrated;
+            errors.push(...result.errors);
+        }
+        return { rowsMigratedMap, errors };
+    }
+}
+const migrateTableLevel = async (levelIndex, tables, tableLevels, tableParallelism, sourceConnection, targetConnection, config, progress) => {
+    Logger.info(`[DataMigrator] Starting table level ${levelIndex + 1}/${tableLevels.length}: ${tables.map((table) => table.name).join(', ')}`);
+    const { largeTables, groups } = groupTablesByLimits(tables);
+    const levelResults = await executeWithConcurrency(largeTables, tableParallelism, async (table) => {
+        return DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
+    });
+    for (const [resultIndex, result] of levelResults.entries()) {
+        const table = largeTables[resultIndex];
+        progress.processedRows += result.rowsMigrated;
+        if (result.errors.length > 0 && table !== undefined) {
+            progress.errors[table.name] = result.errors.join('; ');
+        }
+    }
+    for (const group of groups) {
+        const groupResult = await migrateSmallTablesGroup(group, sourceConnection, targetConnection, config);
+        for (const table of group) {
+            const rowsMigrated = groupResult.rowsMigratedMap[table.name] ?? 0;
+            progress.processedRows += rowsMigrated;
+            if (groupResult.errors.length > 0) {
+                progress.errors[table.name] = groupResult.errors.join('; ');
+            }
+        }
+    }
+};
 const getTableParallelism = (config, targetConnection) => {
     if (targetConnection.type !== 'd1-remote') {
         return 1;
@@ -611,7 +758,42 @@ const createRemoteInsertExecutionQueue = (targetConnection, tableName, rows) =>
     const batchSettings = getInsertBatchSettings(targetConnection);
     return createInsertStatements(targetConnection.type, batchSettings, tableName, rows);
 };
-const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
+const executeRemoteFileBatch = async (targetConnection, tableName, statements) => {
+    const tempFileDir = path.join('.wrangler', 'tmp');
+    const tempFileName = `migration-${tableName}-${Date.now()}-${Math.random().toString(36).substring(7)}.sql`;
+    const tempFilePath = path.join(tempFileDir, tempFileName);
+    try {
+        mkdirSync(tempFileDir, { recursive: true });
+        const sqlContent = statements.map((s) => bindSqlParameters(s.sql, s.parameters)).join(';\n') + ';';
+        writeFileSync(tempFilePath, sqlContent, 'utf8');
+        const executionStartTime = Date.now();
+        WranglerD1.executeSql({
+            dbName: targetConnection.database,
+            isLocal: false,
+            file: tempFilePath,
+        });
+        const durationMs = Date.now() - executionStartTime;
+        try {
+            unlinkSync(tempFilePath);
+        }
+        catch {
+            // Ignore deletion error
+        }
+        const affectedRows = statements.reduce((sum, s) => sum + s.rowCount, 0);
+        return { affectedRows, durationMs, sqlLength: getSqlByteLength(sqlContent) };
+    }
+    catch (error) {
+        try {
+            unlinkSync(tempFilePath);
+        }
+        catch {
+            // Ignore deletion error
+        }
+        Logger.warn(`[DataMigrator] High-speed SQL file insert failed for ${tableName}, falling back to adaptive statement-by-statement execution. Error: ${getErrorMessage(error)}`);
+        return null;
+    }
+};
+async function insertRemoteRowsWithRetry(targetConnection, tableName, data) {
     const pendingRows = [data];
     let insertedRows = 0;
     while (pendingRows.length > 0) {
@@ -621,6 +803,14 @@ const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
         }
         const statements = createRemoteInsertExecutionQueue(targetConnection, tableName, currentRows);
         let shouldRetryCurrentRows = false;
+        if (targetConnection.type === 'd1-remote' && statements.length > 0) {
+            const fileResult = await executeRemoteFileBatch(targetConnection, tableName, statements);
+            if (fileResult !== null) {
+                insertedRows += fileResult.affectedRows;
+                adjustRemoteBatchTuning(targetConnection, fileResult.affectedRows, fileResult.durationMs, fileResult.sqlLength);
+                continue;
+            }
+        }
         for (const [statementIndex, statement] of statements.entries()) {
             try {
                 const { affectedRows, executionDurationMs } = await executeInsertStatement(targetConnection, statement);
@@ -650,7 +840,7 @@ const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
         }
     }
     return insertedRows;
-};
+}
 const createRemoteD1Adapter = (database) => {
     return {
         async connect() {
@@ -688,7 +878,21 @@ const getErrorCause = (error) => {
 };
 const getErrorMessage = (error) => {
     if (error instanceof Error) {
-        return error.message;
+        let msg = error.message;
+        const errObj = error;
+        if (errObj.stderr && (typeof errObj.stderr === 'string' || Buffer.isBuffer(errObj.stderr))) {
+            const stderrStr = errObj.stderr.toString().trim();
+            if (stderrStr) {
+                msg += `\nStderr: ${stderrStr}`;
+            }
+        }
+        if (errObj.stdout && (typeof errObj.stdout === 'string' || Buffer.isBuffer(errObj.stdout))) {
+            const stdoutStr = errObj.stdout.toString().trim();
+            if (stdoutStr) {
+                msg += `\nStdout: ${stdoutStr}`;
+            }
+        }
+        return msg;
     }
     return String(error);
 };
@@ -704,6 +908,61 @@ const getErrorChainMessages = (error) => {
     }
     return [...new Set(messages)];
 };
+const cacheTargetRowCount = (targetConnection, row) => {
+    const name = typeof row['name'] === 'string' ? row['name'] : null;
+    const count = Number(row['count']);
+    if (name !== null && Number.isFinite(count) && targetConnection.targetRowCountsCache) {
+        targetConnection.targetRowCountsCache[name] = count;
+    }
+};
+const cacheTargetRowCountPayload = (targetConnection, payload) => {
+    for (const statementResult of payload) {
+        const row = statementResult.results?.[0];
+        if (row !== undefined) {
+            cacheTargetRowCount(targetConnection, row);
+        }
+    }
+};
+const unlinkTempFile = (tempFilePath) => {
+    try {
+        unlinkSync(tempFilePath);
+    }
+    catch (error) {
+        Logger.debug(`[DataMigrator] Failed to remove temp file ${tempFilePath}: ${getErrorMessage(error)}`);
+    }
+};
+const executeRemoteCountPrefetch = (targetConnection, sqlStatements) => {
+    const tempFileDir = path.join('.wrangler', 'tmp');
+    const tempFileName = `migration-counts-${Date.now()}-${Math.random().toString(36).substring(7)}.sql`;
+    const tempFilePath = path.join(tempFileDir, tempFileName);
+    try {
+        mkdirSync(tempFileDir, { recursive: true });
+        writeFileSync(tempFilePath, `${sqlStatements};`, 'utf8');
+        const output = WranglerD1.executeSql({
+            dbName: targetConnection.database,
+            isLocal: false,
+            file: tempFilePath,
+        });
+        const payload = extractWranglerJson(output);
+        if (payload !== null) {
+            cacheTargetRowCountPayload(targetConnection, payload);
+        }
+    }
+    finally {
+        unlinkTempFile(tempFilePath);
+    }
+};
+const executeLocalCountPrefetch = async (targetConnection, sqlStatements) => {
+    if (!targetConnection.adapter) {
+        return;
+    }
+    for (const sql of sqlStatements) {
+        const result = await targetConnection.adapter.query(sql, []);
+        for (const row of result.rows) {
+            cacheTargetRowCount(targetConnection, row);
+        }
+    }
+};
 const describeDriverError = (error) => {
     if (error === null || typeof error !== 'object') {
         return undefined;
@@ -870,22 +1129,13 @@ export const DataMigrator = Object.freeze({
             Logger.info(`Migrating ${progress.totalTables} tables with ${progress.totalRows} total rows`);
             if (targetConnection.adapter) {
                 await DataMigrator.prepareTargetSchema(sourceConnection, targetConnection, config);
+                await DataMigrator.populateTargetRowCountsCache(targetConnection, schema.tables);
             }
             Logger.info('Starting table migration...');
             const tableLevels = buildTableMigrationLevels(schema.tables);
             const tableParallelism = getTableParallelism(config, targetConnection);
             for (const [levelIndex, tables] of tableLevels.entries()) {
-                Logger.info(`[DataMigrator] Starting table level ${levelIndex + 1}/${tableLevels.length}: ${tables.map((table) => table.name).join(', ')}`);
-                const levelResults = await executeWithConcurrency(tables, tableParallelism, async (table) => {
-                    return DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
-                });
-                for (const [resultIndex, result] of levelResults.entries()) {
-                    const table = tables[resultIndex];
-                    progress.processedRows += result.rowsMigrated;
-                    if (result.errors.length > 0 && table !== undefined) {
-                        progress.errors[table.name] = result.errors.join('; ');
-                    }
-                }
+                await migrateTableLevel(levelIndex, tables, tableLevels, tableParallelism, sourceConnection, targetConnection, config, progress);
             }
             progress.totalRows = Math.max(progress.totalRows, progress.processedRows);
             // Update final percentage
@@ -998,8 +1248,15 @@ export const DataMigrator = Object.freeze({
             if (schemaStatements.length === 0) {
                 return;
             }
-            const batchSql = `${schemaStatements.join(';\n')};`;
-            await adapter.query(batchSql, []);
+            if (targetConnection.type === 'd1-remote') {
+                const batchSql = `${schemaStatements.join(';\n')};`;
+                await adapter.query(batchSql, []);
+            }
+            else {
+                for (const statement of schemaStatements) {
+                    await adapter.query(statement, []);
+                }
+            }
             schemaStatements.length = 0;
         };
         const pushSchemaStatement = async (sql) => {
@@ -1050,12 +1307,43 @@ export const DataMigrator = Object.freeze({
         Logger.info(`Found ${tables.length} tables`);
         return { tables };
     },
+    /**
+     * Populate target table row counts cache to optimize performance and prevent countless individual DB count queries
+     */
+    async populateTargetRowCountsCache(targetConnection, tables) {
+        if (!targetConnection.adapter)
+            return;
+        targetConnection.targetRowCountsCache = {};
+        if (tables.length === 0)
+            return;
+        Logger.info(`[DataMigrator] Pre-fetching target row counts for ${tables.length} tables to optimize migration speed...`);
+        const chunkSize = 50;
+        for (let i = 0; i < tables.length; i += chunkSize) {
+            const chunk = tables.slice(i, i + chunkSize);
+            const countQueries = chunk.map((t) => `SELECT '${t.name.replaceAll("'", "''")}' as name, COUNT(*) as count FROM \`${t.name}\``);
+            try {
+                if (targetConnection.type === 'd1-remote') {
+                    executeRemoteCountPrefetch(targetConnection, countQueries.join(';\n'));
+                }
+                else {
+                    await executeLocalCountPrefetch(targetConnection, countQueries);
+                }
+            }
+            catch (error) {
+                Logger.debug(`[DataMigrator] Chunked pre-fetch failed, falling back to individual table counts on demand. Error: ${getErrorMessage(error)}`);
+            }
+        }
+    },
     /**
      * Get target table row count for resumability
      */
     async getTargetRowCount(targetConnection, tableName) {
         if (!targetConnection.adapter)
             return 0;
+        if (targetConnection.targetRowCountsCache &&
+            typeof targetConnection.targetRowCountsCache[tableName] === 'number') {
+            return targetConnection.targetRowCountsCache[tableName];
+        }
         try {
             const result = await targetConnection.adapter.query(`SELECT COUNT(*) as count FROM \`${tableName}\``, []);
             const count = result.rows[0]?.['count'];
@@ -1206,7 +1494,12 @@ export const DataMigrator = Object.freeze({
             throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
         }
         if (targetConnection.type === 'd1-remote') {
-            return insertRemoteRowsWithRetry(targetConnection, tableName, data);
+            const insertedRows = await insertRemoteRowsWithRetry(targetConnection, tableName, data);
+            if (targetConnection.targetRowCountsCache &&
+                typeof targetConnection.targetRowCountsCache[tableName] === 'number') {
+                targetConnection.targetRowCountsCache[tableName] += insertedRows;
+            }
+            return insertedRows;
         }
         const batchSettings = getInsertBatchSettings(targetConnection);
         const statements = createInsertStatements(targetConnection.type, batchSettings, tableName, data);
@@ -1217,6 +1510,10 @@ export const DataMigrator = Object.freeze({
             insertedRows += affectedRows;
             adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, getSqlByteLength(statement.sql));
         }
+        if (targetConnection.targetRowCountsCache &&
+            typeof targetConnection.targetRowCountsCache[tableName] === 'number') {
+            targetConnection.targetRowCountsCache[tableName] += insertedRows;
+        }
         return insertedRows;
     },
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zintrust/d1-migrator",
-  "version": "2.0.3",
+  "version": "2.0.4",
   "description": "Resumable database migration toolkit for moving data to Cloudflare D1 with ZinTrust.",
   "private": false,
   "type": "module",