@zintrust/d1-migrator 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +871 -0
- package/dist/cli/DataMigrator.d.ts +104 -0
- package/dist/cli/DataMigrator.d.ts.map +1 -0
- package/dist/cli/DataMigrator.js +431 -0
- package/dist/cli/MigrateToD1Command.d.ts +52 -0
- package/dist/cli/MigrateToD1Command.d.ts.map +1 -0
- package/dist/cli/MigrateToD1Command.js +600 -0
- package/dist/cli/ProgressTracker.d.ts +32 -0
- package/dist/cli/ProgressTracker.d.ts.map +1 -0
- package/dist/cli/ProgressTracker.js +95 -0
- package/dist/cli/SchemaAnalyzer.d.ts +130 -0
- package/dist/cli/SchemaAnalyzer.d.ts.map +1 -0
- package/dist/cli/SchemaAnalyzer.js +660 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +32 -0
- package/dist/schema/SchemaBuilder.d.ts +51 -0
- package/dist/schema/SchemaBuilder.d.ts.map +1 -0
- package/dist/schema/SchemaBuilder.js +165 -0
- package/dist/schema/TypeConverter.d.ts +35 -0
- package/dist/schema/TypeConverter.d.ts.map +1 -0
- package/dist/schema/TypeConverter.js +187 -0
- package/dist/schema/Validator.d.ts +74 -0
- package/dist/schema/Validator.d.ts.map +1 -0
- package/dist/schema/Validator.js +225 -0
- package/dist/types.d.ts +145 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/dist/utils/CheckpointManager.d.ts +48 -0
- package/dist/utils/CheckpointManager.d.ts.map +1 -0
- package/dist/utils/CheckpointManager.js +191 -0
- package/dist/utils/DataValidator.d.ts +46 -0
- package/dist/utils/DataValidator.d.ts.map +1 -0
- package/dist/utils/DataValidator.js +139 -0
- package/package.json +37 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Data Migrator
|
|
3
|
+
* Handles the actual data migration between databases
|
|
4
|
+
*/
|
|
5
|
+
import type { MigrationConfig, MigrationProgress } from '../types';
|
|
6
|
+
/**
|
|
7
|
+
* Database connection types
|
|
8
|
+
*/
|
|
9
|
+
export interface SourceConnection {
|
|
10
|
+
driver: MigrationConfig['sourceDriver'];
|
|
11
|
+
connectionString: string;
|
|
12
|
+
connected: boolean;
|
|
13
|
+
adapter?: DatabaseAdapter;
|
|
14
|
+
}
|
|
15
|
+
export interface TargetConnection {
|
|
16
|
+
type: 'd1' | 'd1-remote';
|
|
17
|
+
database: string;
|
|
18
|
+
connected: boolean;
|
|
19
|
+
adapter?: DatabaseAdapter;
|
|
20
|
+
}
|
|
21
|
+
export interface TableInfo {
|
|
22
|
+
name: string;
|
|
23
|
+
rowCount?: number;
|
|
24
|
+
}
|
|
25
|
+
type AdapterQueryResult = {
|
|
26
|
+
rows: Record<string, unknown>[];
|
|
27
|
+
rowCount?: number;
|
|
28
|
+
};
|
|
29
|
+
type DatabaseAdapter = {
|
|
30
|
+
connect(): Promise<void>;
|
|
31
|
+
disconnect?(): Promise<void>;
|
|
32
|
+
query(sql: string, parameters: unknown[]): Promise<AdapterQueryResult>;
|
|
33
|
+
};
|
|
34
|
+
type MigrationVerificationError = {
|
|
35
|
+
table: string;
|
|
36
|
+
offset: number;
|
|
37
|
+
expectedRows: number;
|
|
38
|
+
insertedRows: number;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* DataMigrator - Sealed namespace for data migration
|
|
42
|
+
* Provides chunked data migration with progress tracking
|
|
43
|
+
*/
|
|
44
|
+
export declare const DataMigrator: Readonly<{
|
|
45
|
+
/**
|
|
46
|
+
* Migrate data from source to target
|
|
47
|
+
*/
|
|
48
|
+
migrateData(config: MigrationConfig): Promise<MigrationProgress>;
|
|
49
|
+
/**
|
|
50
|
+
* Connect to source database
|
|
51
|
+
*/
|
|
52
|
+
connectToSource(config: MigrationConfig): Promise<SourceConnection>;
|
|
53
|
+
/**
|
|
54
|
+
* Connect to target D1 database
|
|
55
|
+
*/
|
|
56
|
+
connectToTarget(config: MigrationConfig): Promise<TargetConnection>;
|
|
57
|
+
/**
|
|
58
|
+
* Prepare target schema using source structure
|
|
59
|
+
*/
|
|
60
|
+
prepareTargetSchema(sourceConnection: SourceConnection, targetConnection: TargetConnection, config: MigrationConfig): Promise<void>;
|
|
61
|
+
/**
|
|
62
|
+
* Get schema information from source database
|
|
63
|
+
*/
|
|
64
|
+
getSchemaInfo(_connection: SourceConnection): Promise<{
|
|
65
|
+
tables: TableInfo[];
|
|
66
|
+
}>;
|
|
67
|
+
/**
|
|
68
|
+
* Migrate single table
|
|
69
|
+
*/
|
|
70
|
+
migrateTable(table: TableInfo, sourceConnection: SourceConnection, targetConnection: TargetConnection, config: MigrationConfig): Promise<{
|
|
71
|
+
rowsMigrated: number;
|
|
72
|
+
errors: string[];
|
|
73
|
+
}>;
|
|
74
|
+
/**
|
|
75
|
+
* Read data chunk from source database
|
|
76
|
+
*/
|
|
77
|
+
readDataChunk(sourceConnection: SourceConnection, tableName: string, offset: number, batchSize: number): Promise<Record<string, unknown>[]>;
|
|
78
|
+
/**
|
|
79
|
+
* Transform data for D1 compatibility
|
|
80
|
+
*/
|
|
81
|
+
transformData(chunk: Record<string, unknown>[], tableName: string): Promise<Record<string, unknown>[]>;
|
|
82
|
+
/**
|
|
83
|
+
* Insert data into target database
|
|
84
|
+
*/
|
|
85
|
+
insertData(targetConnection: TargetConnection, tableName: string, data: Record<string, unknown>[]): Promise<number>;
|
|
86
|
+
/**
|
|
87
|
+
* Build chunked SELECT SQL by source driver
|
|
88
|
+
*/
|
|
89
|
+
buildSelectChunkSQL(driver: MigrationConfig["sourceDriver"], tableName: string): string;
|
|
90
|
+
/**
|
|
91
|
+
* Build chunk verification error object
|
|
92
|
+
*/
|
|
93
|
+
createChunkVerificationError(table: string, offset: number, expectedRows: number, insertedRows: number): MigrationVerificationError;
|
|
94
|
+
/**
|
|
95
|
+
* Create migration progress tracker
|
|
96
|
+
*/
|
|
97
|
+
createProgress(migrationId: string): MigrationProgress;
|
|
98
|
+
/**
|
|
99
|
+
* Update migration progress
|
|
100
|
+
*/
|
|
101
|
+
updateProgress(progress: MigrationProgress, updates: Partial<MigrationProgress>): MigrationProgress;
|
|
102
|
+
}>;
|
|
103
|
+
export {};
|
|
104
|
+
//# sourceMappingURL=DataMigrator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAiEF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoEzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAyBzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAmChB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAiBpF;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAyEtD;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAkBrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,aACrB,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA4CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAkClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
/* eslint-disable no-await-in-loop */
|
|
2
|
+
/**
|
|
3
|
+
* Data Migrator
|
|
4
|
+
* Handles the actual data migration between databases
|
|
5
|
+
*/
|
|
6
|
+
import { ErrorFactory, Logger } from '@zintrust/core';
|
|
7
|
+
import { MySQLAdapter } from '@zintrust/db-mysql';
|
|
8
|
+
import { PostgreSQLAdapter } from '@zintrust/db-postgres';
|
|
9
|
+
import { SQLiteAdapter } from '@zintrust/db-sqlite';
|
|
10
|
+
import { SQLServerAdapter } from '@zintrust/db-sqlserver';
|
|
11
|
+
import { SchemaBuilder } from '../schema/SchemaBuilder';
|
|
12
|
+
import { SchemaAnalyzer } from './SchemaAnalyzer';
|
|
13
|
+
const parseConnectionDetails = (connectionString, defaultPort, defaultDatabase, defaultUsername) => {
|
|
14
|
+
try {
|
|
15
|
+
const parsed = new URL(connectionString);
|
|
16
|
+
const databaseName = decodeURIComponent(parsed.pathname.replace(/^\/+/, ''));
|
|
17
|
+
return {
|
|
18
|
+
host: parsed.hostname || 'localhost',
|
|
19
|
+
port: parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort,
|
|
20
|
+
database: databaseName || defaultDatabase,
|
|
21
|
+
username: parsed.username ? decodeURIComponent(parsed.username) : defaultUsername,
|
|
22
|
+
password: parsed.password ? decodeURIComponent(parsed.password) : '',
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
throw ErrorFactory.createValidationError('Invalid source connection string format', error);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const parseSqliteDatabasePath = (connectionString) => {
|
|
30
|
+
const trimmed = connectionString.trim();
|
|
31
|
+
if (trimmed.length === 0) {
|
|
32
|
+
return ':memory:';
|
|
33
|
+
}
|
|
34
|
+
if (!trimmed.includes('://')) {
|
|
35
|
+
return trimmed;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
const parsed = new URL(trimmed);
|
|
39
|
+
if (parsed.protocol !== 'sqlite:') {
|
|
40
|
+
return trimmed;
|
|
41
|
+
}
|
|
42
|
+
const pathName = decodeURIComponent(parsed.pathname);
|
|
43
|
+
return pathName.length > 0 ? pathName : ':memory:';
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return trimmed;
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
const safelyDisconnect = async (label, connection) => {
|
|
50
|
+
try {
|
|
51
|
+
await connection?.adapter?.disconnect?.();
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
Logger.warn(`Failed to close ${label} adapter: ${error}`);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* DataMigrator - Sealed namespace for data migration
|
|
59
|
+
* Provides chunked data migration with progress tracking
|
|
60
|
+
*/
|
|
61
|
+
export const DataMigrator = Object.freeze({
|
|
62
|
+
/**
|
|
63
|
+
* Migrate data from source to target
|
|
64
|
+
*/
|
|
65
|
+
async migrateData(config) {
|
|
66
|
+
Logger.info('Starting data migration...');
|
|
67
|
+
let sourceConnection = null;
|
|
68
|
+
let targetConnection = null;
|
|
69
|
+
try {
|
|
70
|
+
// Initialize progress tracking
|
|
71
|
+
const progress = {
|
|
72
|
+
migrationId: config.migrationId || 'unknown',
|
|
73
|
+
startTime: new Date(),
|
|
74
|
+
currentTable: '',
|
|
75
|
+
table: '',
|
|
76
|
+
totalTables: 0,
|
|
77
|
+
processedRows: 0,
|
|
78
|
+
totalRows: 0,
|
|
79
|
+
percentage: 0,
|
|
80
|
+
errors: {},
|
|
81
|
+
status: 'processing',
|
|
82
|
+
};
|
|
83
|
+
// Connect to source database
|
|
84
|
+
Logger.info('Connecting to source database...');
|
|
85
|
+
sourceConnection = await DataMigrator.connectToSource(config);
|
|
86
|
+
// Connect to target D1 database
|
|
87
|
+
Logger.info('Connecting to target D1 database...');
|
|
88
|
+
targetConnection = await DataMigrator.connectToTarget(config);
|
|
89
|
+
// Get schema information
|
|
90
|
+
const schema = await DataMigrator.getSchemaInfo(sourceConnection);
|
|
91
|
+
progress.totalTables = schema.tables.length;
|
|
92
|
+
// Calculate total rows for progress tracking
|
|
93
|
+
progress.totalRows = schema.tables.reduce((total, table) => total + (table.rowCount || 0), 0);
|
|
94
|
+
Logger.info(`Migrating ${progress.totalTables} tables with ${progress.totalRows} total rows`);
|
|
95
|
+
if (targetConnection.adapter) {
|
|
96
|
+
await DataMigrator.prepareTargetSchema(sourceConnection, targetConnection, config);
|
|
97
|
+
}
|
|
98
|
+
// Migrate each table sequentially for reliable D1/SQLite writes
|
|
99
|
+
Logger.info('Starting table migration...');
|
|
100
|
+
for (const table of schema.tables) {
|
|
101
|
+
Logger.info(`Migrating table: ${table.name}`);
|
|
102
|
+
const result = await DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
|
|
103
|
+
progress.processedRows += result.rowsMigrated;
|
|
104
|
+
// Add any errors to progress
|
|
105
|
+
if (result.errors.length > 0) {
|
|
106
|
+
progress.errors[table.name] = result.errors.join('; ');
|
|
107
|
+
}
|
|
108
|
+
Logger.info(`Table ${table.name} completed: ${result.rowsMigrated} rows migrated`);
|
|
109
|
+
}
|
|
110
|
+
// Update final percentage
|
|
111
|
+
progress.percentage =
|
|
112
|
+
progress.totalRows > 0
|
|
113
|
+
? Math.round((progress.processedRows / progress.totalRows) * 100)
|
|
114
|
+
: 0;
|
|
115
|
+
progress.status = Object.keys(progress.errors).length > 0 ? 'failed' : 'completed';
|
|
116
|
+
Logger.info(`Migration completed: ${progress.processedRows}/${progress.totalRows} rows migrated`);
|
|
117
|
+
return progress;
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
Logger.error('Data migration failed:', error);
|
|
121
|
+
throw error;
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
await safelyDisconnect('source', sourceConnection);
|
|
125
|
+
await safelyDisconnect('target', targetConnection);
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
/**
|
|
129
|
+
* Connect to source database
|
|
130
|
+
*/
|
|
131
|
+
async connectToSource(config) {
|
|
132
|
+
Logger.info(`Connecting to ${config.sourceDriver} database: ${config.sourceConnection}`);
|
|
133
|
+
let adapter;
|
|
134
|
+
switch (config.sourceDriver) {
|
|
135
|
+
case 'mysql':
|
|
136
|
+
adapter = MySQLAdapter.create({
|
|
137
|
+
driver: 'mysql',
|
|
138
|
+
connectionString: config.sourceConnection,
|
|
139
|
+
});
|
|
140
|
+
break;
|
|
141
|
+
case 'postgresql': {
|
|
142
|
+
const connectionDetails = parseConnectionDetails(config.sourceConnection, 5432, 'postgres', 'postgres');
|
|
143
|
+
adapter = PostgreSQLAdapter.create({
|
|
144
|
+
driver: 'postgresql',
|
|
145
|
+
host: connectionDetails.host,
|
|
146
|
+
port: connectionDetails.port,
|
|
147
|
+
database: connectionDetails.database,
|
|
148
|
+
username: connectionDetails.username,
|
|
149
|
+
password: connectionDetails.password,
|
|
150
|
+
});
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
case 'sqlite':
|
|
154
|
+
adapter = SQLiteAdapter.create({
|
|
155
|
+
driver: 'sqlite',
|
|
156
|
+
database: parseSqliteDatabasePath(config.sourceConnection),
|
|
157
|
+
});
|
|
158
|
+
break;
|
|
159
|
+
case 'sqlserver': {
|
|
160
|
+
const connectionDetails = parseConnectionDetails(config.sourceConnection, 1433, 'master', 'sa');
|
|
161
|
+
adapter = SQLServerAdapter.create({
|
|
162
|
+
driver: 'sqlserver',
|
|
163
|
+
host: connectionDetails.host,
|
|
164
|
+
port: connectionDetails.port,
|
|
165
|
+
database: connectionDetails.database,
|
|
166
|
+
username: connectionDetails.username,
|
|
167
|
+
password: connectionDetails.password,
|
|
168
|
+
});
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
default:
|
|
172
|
+
throw ErrorFactory.createValidationError(`Unsupported driver: ${config.sourceDriver}`);
|
|
173
|
+
}
|
|
174
|
+
await adapter.connect();
|
|
175
|
+
const connection = {
|
|
176
|
+
driver: config.sourceDriver,
|
|
177
|
+
connectionString: config.sourceConnection || '',
|
|
178
|
+
connected: true,
|
|
179
|
+
adapter,
|
|
180
|
+
};
|
|
181
|
+
Logger.info('✓ Source database connected');
|
|
182
|
+
return connection;
|
|
183
|
+
},
|
|
184
|
+
/**
|
|
185
|
+
* Connect to target D1 database
|
|
186
|
+
*/
|
|
187
|
+
async connectToTarget(config) {
|
|
188
|
+
Logger.info(`Connecting to target D1 database: ${config.targetDatabase}`);
|
|
189
|
+
const connection = {
|
|
190
|
+
type: config.targetType,
|
|
191
|
+
database: config.targetDatabase,
|
|
192
|
+
connected: true,
|
|
193
|
+
};
|
|
194
|
+
if (config.targetType === 'd1') {
|
|
195
|
+
const d1LocalPath = `.wrangler/state/v3/d1/${config.targetDatabase}/db.sqlite`;
|
|
196
|
+
const d1Local = SQLiteAdapter.create({ driver: 'sqlite', database: d1LocalPath });
|
|
197
|
+
try {
|
|
198
|
+
await d1Local.connect();
|
|
199
|
+
connection.adapter = d1Local;
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
Logger.warn(`Unable to connect local D1 path ${d1LocalPath}: ${error}`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
Logger.info('✓ Target D1 database connected');
|
|
206
|
+
return connection;
|
|
207
|
+
},
|
|
208
|
+
/**
|
|
209
|
+
* Prepare target schema using source structure
|
|
210
|
+
*/
|
|
211
|
+
async prepareTargetSchema(sourceConnection, targetConnection, config) {
|
|
212
|
+
if (!targetConnection.adapter) {
|
|
213
|
+
Logger.warn('No target adapter available; skipping schema preparation');
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
Logger.info('Preparing target D1 schema...');
|
|
217
|
+
const sourceSchema = await SchemaAnalyzer.analyzeSchema({
|
|
218
|
+
driver: sourceConnection.driver,
|
|
219
|
+
connectionString: sourceConnection.connectionString,
|
|
220
|
+
});
|
|
221
|
+
const d1Schema = SchemaBuilder.buildD1Schema(sourceSchema.tables, config.sourceDriver);
|
|
222
|
+
for (const table of d1Schema) {
|
|
223
|
+
const createSQL = SchemaBuilder.generateCreateTableSQL(table).replace(/^CREATE TABLE\s+/i, 'CREATE TABLE IF NOT EXISTS ');
|
|
224
|
+
await targetConnection.adapter.query(createSQL, []);
|
|
225
|
+
const indexSQL = SchemaBuilder.generateIndexSQL(table).map((sql) => sql
|
|
226
|
+
.replace(/^CREATE\s+UNIQUE\s+INDEX\s+/i, 'CREATE UNIQUE INDEX IF NOT EXISTS ')
|
|
227
|
+
.replace(/^CREATE\s+INDEX\s+/i, 'CREATE INDEX IF NOT EXISTS '));
|
|
228
|
+
for (const sql of indexSQL) {
|
|
229
|
+
await targetConnection.adapter.query(sql, []);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
Logger.info(`✓ Target schema prepared for ${d1Schema.length} tables`);
|
|
233
|
+
},
|
|
234
|
+
/**
|
|
235
|
+
* Get schema information from source database
|
|
236
|
+
*/
|
|
237
|
+
async getSchemaInfo(_connection) {
|
|
238
|
+
Logger.info('Retrieving schema information...');
|
|
239
|
+
const sourceSchema = await SchemaAnalyzer.analyzeSchema({
|
|
240
|
+
driver: _connection.driver,
|
|
241
|
+
connectionString: _connection.connectionString,
|
|
242
|
+
});
|
|
243
|
+
const tables = sourceSchema.tables.map((table) => ({
|
|
244
|
+
name: table.name,
|
|
245
|
+
rowCount: table.rowCount || 0,
|
|
246
|
+
}));
|
|
247
|
+
Logger.info(`Found ${tables.length} tables`);
|
|
248
|
+
return { tables };
|
|
249
|
+
},
|
|
250
|
+
/**
|
|
251
|
+
* Migrate single table
|
|
252
|
+
*/
|
|
253
|
+
async migrateTable(table, sourceConnection, targetConnection, config) {
|
|
254
|
+
Logger.info(`Migrating table: ${table.name}`);
|
|
255
|
+
const errors = [];
|
|
256
|
+
let rowsMigrated = 0;
|
|
257
|
+
try {
|
|
258
|
+
const totalRows = table.rowCount || 0;
|
|
259
|
+
const batchSize = config.batchSize || 1000;
|
|
260
|
+
Logger.info(`Processing ${totalRows} rows in batches of ${batchSize}`);
|
|
261
|
+
// Process data in chunks sequentially for data integrity
|
|
262
|
+
for (let offset = 0; offset < totalRows; offset += batchSize) {
|
|
263
|
+
try {
|
|
264
|
+
const chunk = await DataMigrator.readDataChunk(sourceConnection, table.name, offset, batchSize);
|
|
265
|
+
if (chunk.length === 0)
|
|
266
|
+
break;
|
|
267
|
+
// Transform data for D1 compatibility
|
|
268
|
+
const transformedChunk = await DataMigrator.transformData(chunk, table.name);
|
|
269
|
+
// Insert data into target
|
|
270
|
+
const insertedRows = await DataMigrator.insertData(targetConnection, table.name, transformedChunk);
|
|
271
|
+
if (insertedRows !== chunk.length) {
|
|
272
|
+
const verificationError = DataMigrator.createChunkVerificationError(table.name, offset, chunk.length, insertedRows);
|
|
273
|
+
throw ErrorFactory.createValidationError(`Chunk insert mismatch on ${table.name}`, verificationError);
|
|
274
|
+
}
|
|
275
|
+
rowsMigrated += insertedRows;
|
|
276
|
+
// Log progress for large tables
|
|
277
|
+
if (totalRows > 10000 && rowsMigrated % (batchSize * 10) === 0) {
|
|
278
|
+
const percentage = Math.round((rowsMigrated / totalRows) * 100);
|
|
279
|
+
Logger.info(`Table ${table.name}: ${rowsMigrated}/${totalRows} (${percentage}%)`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
catch (error) {
|
|
283
|
+
const errorMsg = `Chunk processing failed at offset ${offset}: ${error}`;
|
|
284
|
+
Logger.error(errorMsg);
|
|
285
|
+
errors.push(errorMsg);
|
|
286
|
+
// Continue with next chunk instead of failing completely
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
Logger.info(`Table ${table.name} completed: ${rowsMigrated} rows migrated`);
|
|
291
|
+
}
|
|
292
|
+
catch (error) {
|
|
293
|
+
const errorMsg = `Failed to migrate table ${table.name}: ${error}`;
|
|
294
|
+
Logger.error(errorMsg);
|
|
295
|
+
errors.push(errorMsg);
|
|
296
|
+
}
|
|
297
|
+
return { rowsMigrated, errors };
|
|
298
|
+
},
|
|
299
|
+
/**
|
|
300
|
+
* Read data chunk from source database
|
|
301
|
+
*/
|
|
302
|
+
async readDataChunk(sourceConnection, tableName, offset, batchSize) {
|
|
303
|
+
Logger.debug(`Reading chunk from ${tableName}: offset ${offset}, size ${batchSize}`);
|
|
304
|
+
if (!sourceConnection.adapter)
|
|
305
|
+
return [];
|
|
306
|
+
try {
|
|
307
|
+
const selectSql = DataMigrator.buildSelectChunkSQL(sourceConnection.driver, tableName);
|
|
308
|
+
const result = await sourceConnection.adapter.query(`${selectSql} LIMIT ${batchSize} OFFSET ${offset}`, []);
|
|
309
|
+
return result.rows || [];
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
Logger.error(`Chunk read failed ${error}`);
|
|
313
|
+
return [];
|
|
314
|
+
}
|
|
315
|
+
},
|
|
316
|
+
/**
|
|
317
|
+
* Transform data for D1 compatibility
|
|
318
|
+
*/
|
|
319
|
+
async transformData(chunk, tableName) {
|
|
320
|
+
Logger.debug(`Transforming ${chunk.length} rows for table ${tableName}`);
|
|
321
|
+
return chunk.map((row) => {
|
|
322
|
+
const transformed = {};
|
|
323
|
+
for (const [key, rawValue] of Object.entries(row)) {
|
|
324
|
+
const value = rawValue;
|
|
325
|
+
if (value === undefined) {
|
|
326
|
+
transformed[key] = null;
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
if (value instanceof Date) {
|
|
330
|
+
transformed[key] = value.toISOString();
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
if (typeof value === 'bigint') {
|
|
334
|
+
transformed[key] = value.toString();
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
if (typeof value === 'object' && value !== null) {
|
|
338
|
+
const globalBuffer = globalThis;
|
|
339
|
+
if (globalBuffer.Buffer?.isBuffer(value) === true || value instanceof Uint8Array) {
|
|
340
|
+
transformed[key] = value;
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
transformed[key] = JSON.stringify(value);
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
transformed[key] = value;
|
|
347
|
+
}
|
|
348
|
+
return transformed;
|
|
349
|
+
});
|
|
350
|
+
},
|
|
351
|
+
/**
|
|
352
|
+
* Insert data into target database
|
|
353
|
+
*/
|
|
354
|
+
async insertData(targetConnection, tableName, data) {
|
|
355
|
+
Logger.debug(`Inserting ${data.length} rows into ${tableName}`);
|
|
356
|
+
if (data.length === 0)
|
|
357
|
+
return 0;
|
|
358
|
+
if (!targetConnection.adapter) {
|
|
359
|
+
throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
|
|
360
|
+
}
|
|
361
|
+
const keys = Object.keys(data[0]);
|
|
362
|
+
const columnList = keys.map((key) => `\`${key}\``).join(', ');
|
|
363
|
+
const placeholders = keys.map(() => '?').join(', ');
|
|
364
|
+
const sql = `INSERT INTO \`${tableName}\` (${columnList}) VALUES (${placeholders})`;
|
|
365
|
+
let insertedRows = 0;
|
|
366
|
+
for (const row of data) {
|
|
367
|
+
const values = keys.map((key) => row[key]);
|
|
368
|
+
try {
|
|
369
|
+
await targetConnection.adapter.query(sql, values);
|
|
370
|
+
insertedRows += 1;
|
|
371
|
+
}
|
|
372
|
+
catch (error) {
|
|
373
|
+
throw ErrorFactory.createValidationError(`Insert failed for table ${tableName}`, {
|
|
374
|
+
sql,
|
|
375
|
+
row,
|
|
376
|
+
cause: error,
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
return insertedRows;
|
|
381
|
+
},
|
|
382
|
+
/**
|
|
383
|
+
* Build chunked SELECT SQL by source driver
|
|
384
|
+
*/
|
|
385
|
+
buildSelectChunkSQL(driver, tableName) {
|
|
386
|
+
switch (driver) {
|
|
387
|
+
case 'postgresql':
|
|
388
|
+
return `SELECT * FROM "${tableName}"`;
|
|
389
|
+
case 'sqlserver':
|
|
390
|
+
return `SELECT * FROM [${tableName}]`;
|
|
391
|
+
case 'sqlite':
|
|
392
|
+
case 'mysql':
|
|
393
|
+
default:
|
|
394
|
+
return `SELECT * FROM \`${tableName}\``;
|
|
395
|
+
}
|
|
396
|
+
},
|
|
397
|
+
/**
|
|
398
|
+
* Build chunk verification error object
|
|
399
|
+
*/
|
|
400
|
+
createChunkVerificationError(table, offset, expectedRows, insertedRows) {
|
|
401
|
+
return {
|
|
402
|
+
table,
|
|
403
|
+
offset,
|
|
404
|
+
expectedRows,
|
|
405
|
+
insertedRows,
|
|
406
|
+
};
|
|
407
|
+
},
|
|
408
|
+
/**
|
|
409
|
+
* Create migration progress tracker
|
|
410
|
+
*/
|
|
411
|
+
createProgress(migrationId) {
|
|
412
|
+
return {
|
|
413
|
+
migrationId,
|
|
414
|
+
startTime: new Date(),
|
|
415
|
+
currentTable: '',
|
|
416
|
+
table: '',
|
|
417
|
+
totalTables: 0,
|
|
418
|
+
totalRows: 0,
|
|
419
|
+
processedRows: 0,
|
|
420
|
+
percentage: 0,
|
|
421
|
+
errors: {},
|
|
422
|
+
status: 'pending',
|
|
423
|
+
};
|
|
424
|
+
},
|
|
425
|
+
/**
|
|
426
|
+
* Update migration progress
|
|
427
|
+
*/
|
|
428
|
+
updateProgress(progress, updates) {
|
|
429
|
+
return { ...progress, ...updates };
|
|
430
|
+
},
|
|
431
|
+
});
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migrate to D1 Command
|
|
3
|
+
* CLI command for migrating databases to Cloudflare D1
|
|
4
|
+
*/
|
|
5
|
+
import { type CommandOptions } from '@zintrust/core/cli';
|
|
6
|
+
import type { Command } from 'commander';
|
|
7
|
+
import type { MigrationConfig } from '../types';
|
|
8
|
+
type D1MigratorCommand = {
|
|
9
|
+
[x: string]: unknown;
|
|
10
|
+
name: string;
|
|
11
|
+
description: string;
|
|
12
|
+
verbose?: boolean;
|
|
13
|
+
getCommand(): Command;
|
|
14
|
+
addOptions?: (command: Command) => void;
|
|
15
|
+
execute(options: CommandOptions): void | Promise<void>;
|
|
16
|
+
info(message: string): void;
|
|
17
|
+
success(message: string): void;
|
|
18
|
+
warn(message: string): void;
|
|
19
|
+
debug(message: unknown): void;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* MigrateToD1Command - CLI command for D1 migration
|
|
23
|
+
* Uses BaseCommand factory following ZinTrust patterns
|
|
24
|
+
*/
|
|
25
|
+
export declare const MigrateToD1Command: D1MigratorCommand;
|
|
26
|
+
/**
|
|
27
|
+
* Execute migration process
|
|
28
|
+
*/
|
|
29
|
+
declare function executeMigration(config: MigrationConfig): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Run in interactive mode
|
|
32
|
+
*/
|
|
33
|
+
declare function runInteractiveMode(config: MigrationConfig): Promise<void>;
|
|
34
|
+
/**
|
|
35
|
+
* Run in automated mode
|
|
36
|
+
*/
|
|
37
|
+
declare function runAutomatedMode(config: MigrationConfig): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Validate migration configuration
|
|
40
|
+
*/
|
|
41
|
+
declare function validateConfig(config: MigrationConfig): {
|
|
42
|
+
valid: boolean;
|
|
43
|
+
errors: string[];
|
|
44
|
+
};
|
|
45
|
+
export declare const MigrationExecutor: Readonly<{
|
|
46
|
+
executeMigration: typeof executeMigration;
|
|
47
|
+
runInteractiveMode: typeof runInteractiveMode;
|
|
48
|
+
runAutomatedMode: typeof runAutomatedMode;
|
|
49
|
+
validateConfig: typeof validateConfig;
|
|
50
|
+
}>;
|
|
51
|
+
export {};
|
|
52
|
+
//# sourceMappingURL=MigrateToD1Command.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAOhD,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAubF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,iBAgG/B,CAAC;AAEH;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CActE;AAED;;GAEG;AACH,iBAAe,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGxE;AAED;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAiGtE;AAED;;GAEG;AACH,iBAAS,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrF;AAGD,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC"}
|