@zintrust/d1-migrator 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +871 -0
  2. package/dist/cli/DataMigrator.d.ts +104 -0
  3. package/dist/cli/DataMigrator.d.ts.map +1 -0
  4. package/dist/cli/DataMigrator.js +431 -0
  5. package/dist/cli/MigrateToD1Command.d.ts +52 -0
  6. package/dist/cli/MigrateToD1Command.d.ts.map +1 -0
  7. package/dist/cli/MigrateToD1Command.js +600 -0
  8. package/dist/cli/ProgressTracker.d.ts +32 -0
  9. package/dist/cli/ProgressTracker.d.ts.map +1 -0
  10. package/dist/cli/ProgressTracker.js +95 -0
  11. package/dist/cli/SchemaAnalyzer.d.ts +130 -0
  12. package/dist/cli/SchemaAnalyzer.d.ts.map +1 -0
  13. package/dist/cli/SchemaAnalyzer.js +660 -0
  14. package/dist/index.d.ts +46 -0
  15. package/dist/index.d.ts.map +1 -0
  16. package/dist/index.js +32 -0
  17. package/dist/schema/SchemaBuilder.d.ts +51 -0
  18. package/dist/schema/SchemaBuilder.d.ts.map +1 -0
  19. package/dist/schema/SchemaBuilder.js +165 -0
  20. package/dist/schema/TypeConverter.d.ts +35 -0
  21. package/dist/schema/TypeConverter.d.ts.map +1 -0
  22. package/dist/schema/TypeConverter.js +187 -0
  23. package/dist/schema/Validator.d.ts +74 -0
  24. package/dist/schema/Validator.d.ts.map +1 -0
  25. package/dist/schema/Validator.js +225 -0
  26. package/dist/types.d.ts +145 -0
  27. package/dist/types.d.ts.map +1 -0
  28. package/dist/types.js +5 -0
  29. package/dist/utils/CheckpointManager.d.ts +48 -0
  30. package/dist/utils/CheckpointManager.d.ts.map +1 -0
  31. package/dist/utils/CheckpointManager.js +191 -0
  32. package/dist/utils/DataValidator.d.ts +46 -0
  33. package/dist/utils/DataValidator.d.ts.map +1 -0
  34. package/dist/utils/DataValidator.js +139 -0
  35. package/package.json +37 -0
@@ -0,0 +1,104 @@
1
+ /**
2
+ * Data Migrator
3
+ * Handles the actual data migration between databases
4
+ */
5
+ import type { MigrationConfig, MigrationProgress } from '../types';
6
+ /**
7
+ * Database connection types
8
+ */
9
+ export interface SourceConnection {
10
+ driver: MigrationConfig['sourceDriver'];
11
+ connectionString: string;
12
+ connected: boolean;
13
+ adapter?: DatabaseAdapter;
14
+ }
15
+ export interface TargetConnection {
16
+ type: 'd1' | 'd1-remote';
17
+ database: string;
18
+ connected: boolean;
19
+ adapter?: DatabaseAdapter;
20
+ }
21
+ export interface TableInfo {
22
+ name: string;
23
+ rowCount?: number;
24
+ }
25
+ type AdapterQueryResult = {
26
+ rows: Record<string, unknown>[];
27
+ rowCount?: number;
28
+ };
29
+ type DatabaseAdapter = {
30
+ connect(): Promise<void>;
31
+ disconnect?(): Promise<void>;
32
+ query(sql: string, parameters: unknown[]): Promise<AdapterQueryResult>;
33
+ };
34
+ type MigrationVerificationError = {
35
+ table: string;
36
+ offset: number;
37
+ expectedRows: number;
38
+ insertedRows: number;
39
+ };
40
+ /**
41
+ * DataMigrator - Sealed namespace for data migration
42
+ * Provides chunked data migration with progress tracking
43
+ */
44
+ export declare const DataMigrator: Readonly<{
45
+ /**
46
+ * Migrate data from source to target
47
+ */
48
+ migrateData(config: MigrationConfig): Promise<MigrationProgress>;
49
+ /**
50
+ * Connect to source database
51
+ */
52
+ connectToSource(config: MigrationConfig): Promise<SourceConnection>;
53
+ /**
54
+ * Connect to target D1 database
55
+ */
56
+ connectToTarget(config: MigrationConfig): Promise<TargetConnection>;
57
+ /**
58
+ * Prepare target schema using source structure
59
+ */
60
+ prepareTargetSchema(sourceConnection: SourceConnection, targetConnection: TargetConnection, config: MigrationConfig): Promise<void>;
61
+ /**
62
+ * Get schema information from source database
63
+ */
64
+ getSchemaInfo(_connection: SourceConnection): Promise<{
65
+ tables: TableInfo[];
66
+ }>;
67
+ /**
68
+ * Migrate single table
69
+ */
70
+ migrateTable(table: TableInfo, sourceConnection: SourceConnection, targetConnection: TargetConnection, config: MigrationConfig): Promise<{
71
+ rowsMigrated: number;
72
+ errors: string[];
73
+ }>;
74
+ /**
75
+ * Read data chunk from source database
76
+ */
77
+ readDataChunk(sourceConnection: SourceConnection, tableName: string, offset: number, batchSize: number): Promise<Record<string, unknown>[]>;
78
+ /**
79
+ * Transform data for D1 compatibility
80
+ */
81
+ transformData(chunk: Record<string, unknown>[], tableName: string): Promise<Record<string, unknown>[]>;
82
+ /**
83
+ * Insert data into target database
84
+ */
85
+ insertData(targetConnection: TargetConnection, tableName: string, data: Record<string, unknown>[]): Promise<number>;
86
+ /**
87
+ * Build chunked SELECT SQL by source driver
88
+ */
89
+ buildSelectChunkSQL(driver: MigrationConfig["sourceDriver"], tableName: string): string;
90
+ /**
91
+ * Build chunk verification error object
92
+ */
93
+ createChunkVerificationError(table: string, offset: number, expectedRows: number, insertedRows: number): MigrationVerificationError;
94
+ /**
95
+ * Create migration progress tracker
96
+ */
97
+ createProgress(migrationId: string): MigrationProgress;
98
+ /**
99
+ * Update migration progress
100
+ */
101
+ updateProgress(progress: MigrationProgress, updates: Partial<MigrationProgress>): MigrationProgress;
102
+ }>;
103
+ export {};
104
+ //# sourceMappingURL=DataMigrator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAiEF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoEzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAyBzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAmChB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAiBpF;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAyEtD;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAkBrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,aACrB,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA4CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAkClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
@@ -0,0 +1,431 @@
1
+ /* eslint-disable no-await-in-loop */
2
+ /**
3
+ * Data Migrator
4
+ * Handles the actual data migration between databases
5
+ */
6
+ import { ErrorFactory, Logger } from '@zintrust/core';
7
+ import { MySQLAdapter } from '@zintrust/db-mysql';
8
+ import { PostgreSQLAdapter } from '@zintrust/db-postgres';
9
+ import { SQLiteAdapter } from '@zintrust/db-sqlite';
10
+ import { SQLServerAdapter } from '@zintrust/db-sqlserver';
11
+ import { SchemaBuilder } from '../schema/SchemaBuilder';
12
+ import { SchemaAnalyzer } from './SchemaAnalyzer';
13
+ const parseConnectionDetails = (connectionString, defaultPort, defaultDatabase, defaultUsername) => {
14
+ try {
15
+ const parsed = new URL(connectionString);
16
+ const databaseName = decodeURIComponent(parsed.pathname.replace(/^\/+/, ''));
17
+ return {
18
+ host: parsed.hostname || 'localhost',
19
+ port: parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort,
20
+ database: databaseName || defaultDatabase,
21
+ username: parsed.username ? decodeURIComponent(parsed.username) : defaultUsername,
22
+ password: parsed.password ? decodeURIComponent(parsed.password) : '',
23
+ };
24
+ }
25
+ catch (error) {
26
+ throw ErrorFactory.createValidationError('Invalid source connection string format', error);
27
+ }
28
+ };
29
+ const parseSqliteDatabasePath = (connectionString) => {
30
+ const trimmed = connectionString.trim();
31
+ if (trimmed.length === 0) {
32
+ return ':memory:';
33
+ }
34
+ if (!trimmed.includes('://')) {
35
+ return trimmed;
36
+ }
37
+ try {
38
+ const parsed = new URL(trimmed);
39
+ if (parsed.protocol !== 'sqlite:') {
40
+ return trimmed;
41
+ }
42
+ const pathName = decodeURIComponent(parsed.pathname);
43
+ return pathName.length > 0 ? pathName : ':memory:';
44
+ }
45
+ catch {
46
+ return trimmed;
47
+ }
48
+ };
49
+ const safelyDisconnect = async (label, connection) => {
50
+ try {
51
+ await connection?.adapter?.disconnect?.();
52
+ }
53
+ catch (error) {
54
+ Logger.warn(`Failed to close ${label} adapter: ${error}`);
55
+ }
56
+ };
57
+ /**
58
+ * DataMigrator - Sealed namespace for data migration
59
+ * Provides chunked data migration with progress tracking
60
+ */
61
+ export const DataMigrator = Object.freeze({
62
+ /**
63
+ * Migrate data from source to target
64
+ */
65
+ async migrateData(config) {
66
+ Logger.info('Starting data migration...');
67
+ let sourceConnection = null;
68
+ let targetConnection = null;
69
+ try {
70
+ // Initialize progress tracking
71
+ const progress = {
72
+ migrationId: config.migrationId || 'unknown',
73
+ startTime: new Date(),
74
+ currentTable: '',
75
+ table: '',
76
+ totalTables: 0,
77
+ processedRows: 0,
78
+ totalRows: 0,
79
+ percentage: 0,
80
+ errors: {},
81
+ status: 'processing',
82
+ };
83
+ // Connect to source database
84
+ Logger.info('Connecting to source database...');
85
+ sourceConnection = await DataMigrator.connectToSource(config);
86
+ // Connect to target D1 database
87
+ Logger.info('Connecting to target D1 database...');
88
+ targetConnection = await DataMigrator.connectToTarget(config);
89
+ // Get schema information
90
+ const schema = await DataMigrator.getSchemaInfo(sourceConnection);
91
+ progress.totalTables = schema.tables.length;
92
+ // Calculate total rows for progress tracking
93
+ progress.totalRows = schema.tables.reduce((total, table) => total + (table.rowCount || 0), 0);
94
+ Logger.info(`Migrating ${progress.totalTables} tables with ${progress.totalRows} total rows`);
95
+ if (targetConnection.adapter) {
96
+ await DataMigrator.prepareTargetSchema(sourceConnection, targetConnection, config);
97
+ }
98
+ // Migrate each table sequentially for reliable D1/SQLite writes
99
+ Logger.info('Starting table migration...');
100
+ for (const table of schema.tables) {
101
+ Logger.info(`Migrating table: ${table.name}`);
102
+ const result = await DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
103
+ progress.processedRows += result.rowsMigrated;
104
+ // Add any errors to progress
105
+ if (result.errors.length > 0) {
106
+ progress.errors[table.name] = result.errors.join('; ');
107
+ }
108
+ Logger.info(`Table ${table.name} completed: ${result.rowsMigrated} rows migrated`);
109
+ }
110
+ // Update final percentage
111
+ progress.percentage =
112
+ progress.totalRows > 0
113
+ ? Math.round((progress.processedRows / progress.totalRows) * 100)
114
+ : 0;
115
+ progress.status = Object.keys(progress.errors).length > 0 ? 'failed' : 'completed';
116
+ Logger.info(`Migration completed: ${progress.processedRows}/${progress.totalRows} rows migrated`);
117
+ return progress;
118
+ }
119
+ catch (error) {
120
+ Logger.error('Data migration failed:', error);
121
+ throw error;
122
+ }
123
+ finally {
124
+ await safelyDisconnect('source', sourceConnection);
125
+ await safelyDisconnect('target', targetConnection);
126
+ }
127
+ },
128
+ /**
129
+ * Connect to source database
130
+ */
131
+ async connectToSource(config) {
132
+ Logger.info(`Connecting to ${config.sourceDriver} database: ${config.sourceConnection}`);
133
+ let adapter;
134
+ switch (config.sourceDriver) {
135
+ case 'mysql':
136
+ adapter = MySQLAdapter.create({
137
+ driver: 'mysql',
138
+ connectionString: config.sourceConnection,
139
+ });
140
+ break;
141
+ case 'postgresql': {
142
+ const connectionDetails = parseConnectionDetails(config.sourceConnection, 5432, 'postgres', 'postgres');
143
+ adapter = PostgreSQLAdapter.create({
144
+ driver: 'postgresql',
145
+ host: connectionDetails.host,
146
+ port: connectionDetails.port,
147
+ database: connectionDetails.database,
148
+ username: connectionDetails.username,
149
+ password: connectionDetails.password,
150
+ });
151
+ break;
152
+ }
153
+ case 'sqlite':
154
+ adapter = SQLiteAdapter.create({
155
+ driver: 'sqlite',
156
+ database: parseSqliteDatabasePath(config.sourceConnection),
157
+ });
158
+ break;
159
+ case 'sqlserver': {
160
+ const connectionDetails = parseConnectionDetails(config.sourceConnection, 1433, 'master', 'sa');
161
+ adapter = SQLServerAdapter.create({
162
+ driver: 'sqlserver',
163
+ host: connectionDetails.host,
164
+ port: connectionDetails.port,
165
+ database: connectionDetails.database,
166
+ username: connectionDetails.username,
167
+ password: connectionDetails.password,
168
+ });
169
+ break;
170
+ }
171
+ default:
172
+ throw ErrorFactory.createValidationError(`Unsupported driver: ${config.sourceDriver}`);
173
+ }
174
+ await adapter.connect();
175
+ const connection = {
176
+ driver: config.sourceDriver,
177
+ connectionString: config.sourceConnection || '',
178
+ connected: true,
179
+ adapter,
180
+ };
181
+ Logger.info('✓ Source database connected');
182
+ return connection;
183
+ },
184
+ /**
185
+ * Connect to target D1 database
186
+ */
187
+ async connectToTarget(config) {
188
+ Logger.info(`Connecting to target D1 database: ${config.targetDatabase}`);
189
+ const connection = {
190
+ type: config.targetType,
191
+ database: config.targetDatabase,
192
+ connected: true,
193
+ };
194
+ if (config.targetType === 'd1') {
195
+ const d1LocalPath = `.wrangler/state/v3/d1/${config.targetDatabase}/db.sqlite`;
196
+ const d1Local = SQLiteAdapter.create({ driver: 'sqlite', database: d1LocalPath });
197
+ try {
198
+ await d1Local.connect();
199
+ connection.adapter = d1Local;
200
+ }
201
+ catch (error) {
202
+ Logger.warn(`Unable to connect local D1 path ${d1LocalPath}: ${error}`);
203
+ }
204
+ }
205
+ Logger.info('✓ Target D1 database connected');
206
+ return connection;
207
+ },
208
+ /**
209
+ * Prepare target schema using source structure
210
+ */
211
+ async prepareTargetSchema(sourceConnection, targetConnection, config) {
212
+ if (!targetConnection.adapter) {
213
+ Logger.warn('No target adapter available; skipping schema preparation');
214
+ return;
215
+ }
216
+ Logger.info('Preparing target D1 schema...');
217
+ const sourceSchema = await SchemaAnalyzer.analyzeSchema({
218
+ driver: sourceConnection.driver,
219
+ connectionString: sourceConnection.connectionString,
220
+ });
221
+ const d1Schema = SchemaBuilder.buildD1Schema(sourceSchema.tables, config.sourceDriver);
222
+ for (const table of d1Schema) {
223
+ const createSQL = SchemaBuilder.generateCreateTableSQL(table).replace(/^CREATE TABLE\s+/i, 'CREATE TABLE IF NOT EXISTS ');
224
+ await targetConnection.adapter.query(createSQL, []);
225
+ const indexSQL = SchemaBuilder.generateIndexSQL(table).map((sql) => sql
226
+ .replace(/^CREATE\s+UNIQUE\s+INDEX\s+/i, 'CREATE UNIQUE INDEX IF NOT EXISTS ')
227
+ .replace(/^CREATE\s+INDEX\s+/i, 'CREATE INDEX IF NOT EXISTS '));
228
+ for (const sql of indexSQL) {
229
+ await targetConnection.adapter.query(sql, []);
230
+ }
231
+ }
232
+ Logger.info(`✓ Target schema prepared for ${d1Schema.length} tables`);
233
+ },
234
+ /**
235
+ * Get schema information from source database
236
+ */
237
+ async getSchemaInfo(_connection) {
238
+ Logger.info('Retrieving schema information...');
239
+ const sourceSchema = await SchemaAnalyzer.analyzeSchema({
240
+ driver: _connection.driver,
241
+ connectionString: _connection.connectionString,
242
+ });
243
+ const tables = sourceSchema.tables.map((table) => ({
244
+ name: table.name,
245
+ rowCount: table.rowCount || 0,
246
+ }));
247
+ Logger.info(`Found ${tables.length} tables`);
248
+ return { tables };
249
+ },
250
+ /**
251
+ * Migrate single table
252
+ */
253
+ async migrateTable(table, sourceConnection, targetConnection, config) {
254
+ Logger.info(`Migrating table: ${table.name}`);
255
+ const errors = [];
256
+ let rowsMigrated = 0;
257
+ try {
258
+ const totalRows = table.rowCount || 0;
259
+ const batchSize = config.batchSize || 1000;
260
+ Logger.info(`Processing ${totalRows} rows in batches of ${batchSize}`);
261
+ // Process data in chunks sequentially for data integrity
262
+ for (let offset = 0; offset < totalRows; offset += batchSize) {
263
+ try {
264
+ const chunk = await DataMigrator.readDataChunk(sourceConnection, table.name, offset, batchSize);
265
+ if (chunk.length === 0)
266
+ break;
267
+ // Transform data for D1 compatibility
268
+ const transformedChunk = await DataMigrator.transformData(chunk, table.name);
269
+ // Insert data into target
270
+ const insertedRows = await DataMigrator.insertData(targetConnection, table.name, transformedChunk);
271
+ if (insertedRows !== chunk.length) {
272
+ const verificationError = DataMigrator.createChunkVerificationError(table.name, offset, chunk.length, insertedRows);
273
+ throw ErrorFactory.createValidationError(`Chunk insert mismatch on ${table.name}`, verificationError);
274
+ }
275
+ rowsMigrated += insertedRows;
276
+ // Log progress for large tables
277
+ if (totalRows > 10000 && rowsMigrated % (batchSize * 10) === 0) {
278
+ const percentage = Math.round((rowsMigrated / totalRows) * 100);
279
+ Logger.info(`Table ${table.name}: ${rowsMigrated}/${totalRows} (${percentage}%)`);
280
+ }
281
+ }
282
+ catch (error) {
283
+ const errorMsg = `Chunk processing failed at offset ${offset}: ${error}`;
284
+ Logger.error(errorMsg);
285
+ errors.push(errorMsg);
286
+ // Continue with next chunk instead of failing completely
287
+ continue;
288
+ }
289
+ }
290
+ Logger.info(`Table ${table.name} completed: ${rowsMigrated} rows migrated`);
291
+ }
292
+ catch (error) {
293
+ const errorMsg = `Failed to migrate table ${table.name}: ${error}`;
294
+ Logger.error(errorMsg);
295
+ errors.push(errorMsg);
296
+ }
297
+ return { rowsMigrated, errors };
298
+ },
299
+ /**
300
+ * Read data chunk from source database
301
+ */
302
+ async readDataChunk(sourceConnection, tableName, offset, batchSize) {
303
+ Logger.debug(`Reading chunk from ${tableName}: offset ${offset}, size ${batchSize}`);
304
+ if (!sourceConnection.adapter)
305
+ return [];
306
+ try {
307
+ const selectSql = DataMigrator.buildSelectChunkSQL(sourceConnection.driver, tableName);
308
+ const result = await sourceConnection.adapter.query(`${selectSql} LIMIT ${batchSize} OFFSET ${offset}`, []);
309
+ return result.rows || [];
310
+ }
311
+ catch (error) {
312
+ Logger.error(`Chunk read failed ${error}`);
313
+ return [];
314
+ }
315
+ },
316
+ /**
317
+ * Transform data for D1 compatibility
318
+ */
319
+ async transformData(chunk, tableName) {
320
+ Logger.debug(`Transforming ${chunk.length} rows for table ${tableName}`);
321
+ return chunk.map((row) => {
322
+ const transformed = {};
323
+ for (const [key, rawValue] of Object.entries(row)) {
324
+ const value = rawValue;
325
+ if (value === undefined) {
326
+ transformed[key] = null;
327
+ continue;
328
+ }
329
+ if (value instanceof Date) {
330
+ transformed[key] = value.toISOString();
331
+ continue;
332
+ }
333
+ if (typeof value === 'bigint') {
334
+ transformed[key] = value.toString();
335
+ continue;
336
+ }
337
+ if (typeof value === 'object' && value !== null) {
338
+ const globalBuffer = globalThis;
339
+ if (globalBuffer.Buffer?.isBuffer(value) === true || value instanceof Uint8Array) {
340
+ transformed[key] = value;
341
+ continue;
342
+ }
343
+ transformed[key] = JSON.stringify(value);
344
+ continue;
345
+ }
346
+ transformed[key] = value;
347
+ }
348
+ return transformed;
349
+ });
350
+ },
351
+ /**
352
+ * Insert data into target database
353
+ */
354
+ async insertData(targetConnection, tableName, data) {
355
+ Logger.debug(`Inserting ${data.length} rows into ${tableName}`);
356
+ if (data.length === 0)
357
+ return 0;
358
+ if (!targetConnection.adapter) {
359
+ throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
360
+ }
361
+ const keys = Object.keys(data[0]);
362
+ const columnList = keys.map((key) => `\`${key}\``).join(', ');
363
+ const placeholders = keys.map(() => '?').join(', ');
364
+ const sql = `INSERT INTO \`${tableName}\` (${columnList}) VALUES (${placeholders})`;
365
+ let insertedRows = 0;
366
+ for (const row of data) {
367
+ const values = keys.map((key) => row[key]);
368
+ try {
369
+ await targetConnection.adapter.query(sql, values);
370
+ insertedRows += 1;
371
+ }
372
+ catch (error) {
373
+ throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
374
+ sql,
375
+ row,
376
+ cause: error,
377
+ });
378
+ }
379
+ }
380
+ return insertedRows;
381
+ },
382
+ /**
383
+ * Build chunked SELECT SQL by source driver
384
+ */
385
+ buildSelectChunkSQL(driver, tableName) {
386
+ switch (driver) {
387
+ case 'postgresql':
388
+ return `SELECT * FROM "${tableName}"`;
389
+ case 'sqlserver':
390
+ return `SELECT * FROM [${tableName}]`;
391
+ case 'sqlite':
392
+ case 'mysql':
393
+ default:
394
+ return `SELECT * FROM \`${tableName}\``;
395
+ }
396
+ },
397
+ /**
398
+ * Build chunk verification error object
399
+ */
400
+ createChunkVerificationError(table, offset, expectedRows, insertedRows) {
401
+ return {
402
+ table,
403
+ offset,
404
+ expectedRows,
405
+ insertedRows,
406
+ };
407
+ },
408
+ /**
409
+ * Create migration progress tracker
410
+ */
411
+ createProgress(migrationId) {
412
+ return {
413
+ migrationId,
414
+ startTime: new Date(),
415
+ currentTable: '',
416
+ table: '',
417
+ totalTables: 0,
418
+ totalRows: 0,
419
+ processedRows: 0,
420
+ percentage: 0,
421
+ errors: {},
422
+ status: 'pending',
423
+ };
424
+ },
425
+ /**
426
+ * Update migration progress
427
+ */
428
+ updateProgress(progress, updates) {
429
+ return { ...progress, ...updates };
430
+ },
431
+ });
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Migrate to D1 Command
3
+ * CLI command for migrating databases to Cloudflare D1
4
+ */
5
+ import { type CommandOptions } from '@zintrust/core/cli';
6
+ import type { Command } from 'commander';
7
+ import type { MigrationConfig } from '../types';
8
+ type D1MigratorCommand = {
9
+ [x: string]: unknown;
10
+ name: string;
11
+ description: string;
12
+ verbose?: boolean;
13
+ getCommand(): Command;
14
+ addOptions?: (command: Command) => void;
15
+ execute(options: CommandOptions): void | Promise<void>;
16
+ info(message: string): void;
17
+ success(message: string): void;
18
+ warn(message: string): void;
19
+ debug(message: unknown): void;
20
+ };
21
+ /**
22
+ * MigrateToD1Command - CLI command for D1 migration
23
+ * Uses BaseCommand factory following ZinTrust patterns
24
+ */
25
+ export declare const MigrateToD1Command: D1MigratorCommand;
26
+ /**
27
+ * Execute migration process
28
+ */
29
+ declare function executeMigration(config: MigrationConfig): Promise<void>;
30
+ /**
31
+ * Run in interactive mode
32
+ */
33
+ declare function runInteractiveMode(config: MigrationConfig): Promise<void>;
34
+ /**
35
+ * Run in automated mode
36
+ */
37
+ declare function runAutomatedMode(config: MigrationConfig): Promise<void>;
38
+ /**
39
+ * Validate migration configuration
40
+ */
41
+ declare function validateConfig(config: MigrationConfig): {
42
+ valid: boolean;
43
+ errors: string[];
44
+ };
45
+ export declare const MigrationExecutor: Readonly<{
46
+ executeMigration: typeof executeMigration;
47
+ runInteractiveMode: typeof runInteractiveMode;
48
+ runAutomatedMode: typeof runAutomatedMode;
49
+ validateConfig: typeof validateConfig;
50
+ }>;
51
+ export {};
52
+ //# sourceMappingURL=MigrateToD1Command.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAOhD,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAubF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,iBAgG/B,CAAC;AAEH;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CActE;AAED;;GAEG;AACH,iBAAe,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGxE;AAED;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAiGtE;AAED;;GAEG;AACH,iBAAS,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrF;AAGD,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC"}