@zintrust/d1-migrator 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +884 -0
- package/dist/build-manifest.json +14 -14
- package/dist/cli/DataMigrator.d.ts.map +1 -1
- package/dist/cli/DataMigrator.js +9 -7
- package/dist/cli/MigrateToD1Command.js +4 -4
- package/dist/index.js +9 -9
- package/dist/register.js +1 -1
- package/dist/schema/SchemaBuilder.js +2 -2
- package/package.json +14 -3
package/README.md
ADDED
|
@@ -0,0 +1,884 @@
|
|
|
1
|
+
# @zintrust/d1-migrator
|
|
2
|
+
|
|
3
|
+
Migrate any database (MySQL, PostgreSQL, SQLite, SQL Server) to Cloudflare D1 with resumable operations, checkpoint recovery, and comprehensive data validation. Built for production use with careful attention to data integrity and operational safety.
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/@zintrust/d1-migrator)
|
|
6
|
+
[](https://nodejs.org/)
|
|
7
|
+
[](https://www.typescriptlang.org/)
|
|
8
|
+
[](LICENSE.md)
|
|
9
|
+
|
|
10
|
+
> **Reliable, resumable database migrations to Cloudflare D1 with full data integrity verification.**
|
|
11
|
+
|
|
12
|
+
## Features
|
|
13
|
+
|
|
14
|
+
- **Multi-source Support**: Migrate from MySQL, PostgreSQL, SQLite, or SQL Server
|
|
15
|
+
- **Resumable Operations**: Automatic checkpointing allows recovery from failures without data loss
|
|
16
|
+
- **Data Integrity**: Row-count verification, checksums, and validation at every step
|
|
17
|
+
- **D1 Compatibility**: Automatic schema conversion and value transformation for SQLite compatibility
|
|
18
|
+
- **Dry-Run Mode**: Test migrations safely before executing
|
|
19
|
+
- **Interactive Mode**: Get guidance for complex migrations with compatibility issues
|
|
20
|
+
- **Batch Processing**: Configurable batch sizes for memory efficiency
|
|
21
|
+
- **Progress Tracking**: Real-time migration progress with detailed metrics
|
|
22
|
+
- **Error Resilience**: Comprehensive error handling with detailed reporting
|
|
23
|
+
- **Zero Downtime**: Works with live databases without requiring offline periods
|
|
24
|
+
- **TypeScript First**: Full type safety and IDE support out of the box
|
|
25
|
+
|
|
26
|
+
## Requirements
|
|
27
|
+
|
|
28
|
+
- **Node.js**: >= 20.0.0 (ESM support required)
|
|
29
|
+
- **TypeScript**: >= 5.0.0
|
|
30
|
+
- **@zintrust/core**: Latest version
|
|
31
|
+
- **Wrangler**: >= 2.0.0 (for D1 management)
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
Install the package with your preferred package manager:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
npm install @zintrust/d1-migrator
|
|
39
|
+
# or
|
|
40
|
+
yarn add @zintrust/d1-migrator
|
|
41
|
+
# or
|
|
42
|
+
pnpm add @zintrust/d1-migrator
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The package requires source database adapters depending on which database you're migrating from. These are automatically included as dependencies:
|
|
46
|
+
|
|
47
|
+
- MySQL → `@zintrust/db-mysql`
|
|
48
|
+
- PostgreSQL → `@zintrust/db-postgres`
|
|
49
|
+
- SQLite → `@zintrust/db-sqlite`
|
|
50
|
+
- SQL Server → `@zintrust/db-sqlserver`
|
|
51
|
+
- Target → `@zintrust/db-d1`
|
|
52
|
+
|
|
53
|
+
## CLI Auto-Registration
|
|
54
|
+
|
|
55
|
+
No manual registration is required.
|
|
56
|
+
|
|
57
|
+
After installing `@zintrust/d1-migrator`, ZinTrust auto-detects the package and exposes the `zin migrate-to-d1` command automatically:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
npm install @zintrust/d1-migrator
|
|
61
|
+
zin migrate-to-d1 --help
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
You do not need to add `@zintrust/d1-migrator` to `src/zintrust.plugins.ts`, and you do not need to run `zin plugin install` for this package. Installing it is enough.
|
|
65
|
+
|
|
66
|
+
## Quick Start
|
|
67
|
+
|
|
68
|
+
### Via CLI (Recommended)
|
|
69
|
+
|
|
70
|
+
#### Zero-arg command (env-driven)
|
|
71
|
+
|
|
72
|
+
Set env vars once, then run the command without flags:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
export DB_CONNECTION=mysql
|
|
76
|
+
export DB_READ_HOSTS=127.0.0.1
|
|
77
|
+
export DB_PORT=3306
|
|
78
|
+
export DB_DATABASE=zintrust
|
|
79
|
+
export DB_USERNAME=root
|
|
80
|
+
export DB_PASSWORD=secret
|
|
81
|
+
# Optional (defaults to "d1" when omitted)
|
|
82
|
+
export D1_TARGET_DB=zintrust-live-test
|
|
83
|
+
|
|
84
|
+
zin migrate-to-d1
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The command resolves values in this order: **CLI flag → environment variable → built-in default**.
|
|
88
|
+
|
|
89
|
+
#### Explicit flags
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
# Analyze and migrate a MySQL database to D1
|
|
93
|
+
zin migrate-to-d1 \
|
|
94
|
+
--from mysql \
|
|
95
|
+
--source-connection "mysql://user:password@localhost:3306/mydb" \
|
|
96
|
+
--to d1 \
|
|
97
|
+
--target-database my-d1-db
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### Via TypeScript
|
|
101
|
+
|
|
102
|
+
```typescript
|
|
103
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
104
|
+
|
|
105
|
+
const config = {
|
|
106
|
+
sourceConnection: 'mysql://user:password@localhost:3306/mydb',
|
|
107
|
+
sourceDriver: 'mysql',
|
|
108
|
+
targetDatabase: 'my-d1-db',
|
|
109
|
+
targetType: 'd1',
|
|
110
|
+
batchSize: 1000,
|
|
111
|
+
checkpointInterval: 10000,
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const progress = await D1Migrator.DataMigrator.migrateData(config);
|
|
115
|
+
console.log(`Migration complete: ${progress.processedRows} rows migrated`);
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Usage Guide
|
|
119
|
+
|
|
120
|
+
### CLI Commands
|
|
121
|
+
|
|
122
|
+
#### Basic Migration
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
zin migrate-to-d1 \
|
|
126
|
+
--from mysql \
|
|
127
|
+
--source-connection "mysql://user:password@localhost:3306/sourcedb" \
|
|
128
|
+
--to d1 \
|
|
129
|
+
--target-database target-d1-db
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
#### With Custom Batch Size
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
zin migrate-to-d1 \
|
|
136
|
+
--from postgresql \
|
|
137
|
+
--source-connection "postgresql://user:password@localhost:5432/sourcedb" \
|
|
138
|
+
--to d1-remote \
|
|
139
|
+
--target-database my-d1-remote \
|
|
140
|
+
--batch-size 5000 \
|
|
141
|
+
--checkpoint-interval 25000
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
#### Dry Run (Test Mode)
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
zin migrate-to-d1 \
|
|
148
|
+
--from mysql \
|
|
149
|
+
--source-connection "mysql://user:password@localhost:3306/sourcedb" \
|
|
150
|
+
--to d1 \
|
|
151
|
+
--target-database test-d1-db \
|
|
152
|
+
--dry-run
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
#### Schema-Only Analysis
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
zin migrate-to-d1 \
|
|
159
|
+
--from sqlserver \
|
|
160
|
+
--source-connection "mssql://user:password@localhost:1433/sourcedb" \
|
|
161
|
+
--to d1 \
|
|
162
|
+
--target-database target-d1-db \
|
|
163
|
+
--schema-only
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
#### Interactive Mode
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
zin migrate-to-d1 \
|
|
170
|
+
--from mysql \
|
|
171
|
+
--source-connection "mysql://user:password@localhost:3306/sourcedb" \
|
|
172
|
+
--to d1 \
|
|
173
|
+
--target-database target-d1-db \
|
|
174
|
+
--interactive
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
#### Resume Failed Migration
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
zin migrate-to-d1 \
|
|
181
|
+
--resume \
|
|
182
|
+
--migration-id abc123def456
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
### CLI Options
|
|
186
|
+
|
|
187
|
+
| Option | Short | Type | Required | Default | Description |
|
|
188
|
+
| ----------------------- | ----- | ------- | -------- | ------- | ------------------------------------------------------------------ |
|
|
189
|
+
| `--from` | `-f` | string | ✗ | — | Source database type: `mysql`, `postgresql`, `sqlite`, `sqlserver` |
|
|
190
|
+
| `--to` | `-t` | string | ✗ | `d1` | Target: `d1` (local) or `d1-remote` |
|
|
191
|
+
| `--source-connection` | `-s` | string | ✗ | — | Source connection URI (falls back to env or DB\_\* composition) |
|
|
192
|
+
| `--target-database` | `-d` | string | ✗ | `d1` | Target D1 database identifier (or env fallback) |
|
|
193
|
+
| `--batch-size` | `-b` | number | ✗ | `1000` | Records per batch during data copy |
|
|
194
|
+
| `--checkpoint-interval` | `-c` | number | ✗ | `10000` | Save checkpoint every N rows |
|
|
195
|
+
| `--dry-run` | — | boolean | ✗ | `false` | Test migration without making changes |
|
|
196
|
+
| `--schema-only` | — | boolean | ✗ | `false` | Analyze and convert schema only |
|
|
197
|
+
| `--interactive` | `-i` | boolean | ✗ | `false` | Interactive mode for complex migrations |
|
|
198
|
+
| `--resume` | `-r` | boolean | ✗ | `false` | Resume a previously paused/failed migration |
|
|
199
|
+
| `--migration-id` | — | string | ✗ | — | Migration ID to resume (required with `--resume`) |
|
|
200
|
+
|
|
201
|
+
### Environment Variable Fallbacks
|
|
202
|
+
|
|
203
|
+
The command supports env-based execution for all CLI settings.
|
|
204
|
+
|
|
205
|
+
| Setting | Env variables (priority order) |
|
|
206
|
+
| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
|
207
|
+
| Source driver (`--from`) | `MIGRATE_TO_D1_FROM`, `MIGRATE_TO_D1_SOURCE_DRIVER`, `D1_MIGRATOR_SOURCE_DRIVER`, `DB_CONNECTION` |
|
|
208
|
+
| Source URI (`--source-connection`) | `MIGRATE_TO_D1_SOURCE_CONNECTION`, `D1_MIGRATOR_SOURCE_CONNECTION`, `SOURCE_DATABASE_URL`, `DATABASE_URL`, `DB_URL` |
|
|
209
|
+
| Target type (`--to`) | `MIGRATE_TO_D1_TO`, `MIGRATE_TO_D1_TARGET_TYPE`, `D1_MIGRATOR_TARGET_TYPE`, `D1_TARGET_TYPE` |
|
|
210
|
+
| Target DB (`--target-database`) | `MIGRATE_TO_D1_TARGET_DATABASE`, `D1_MIGRATOR_TARGET_DATABASE`, `D1_TARGET_DB`, `D1_DATABASE`, `D1_DATABASE_ID`, `DB_DATABASE` |
|
|
211
|
+
| Batch size (`--batch-size`) | `MIGRATE_TO_D1_BATCH_SIZE`, `D1_MIGRATOR_BATCH_SIZE` |
|
|
212
|
+
| Checkpoint interval (`--checkpoint-interval`) | `MIGRATE_TO_D1_CHECKPOINT_INTERVAL`, `D1_MIGRATOR_CHECKPOINT_INTERVAL` |
|
|
213
|
+
| Dry run (`--dry-run`) | `MIGRATE_TO_D1_DRY_RUN`, `D1_MIGRATOR_DRY_RUN` |
|
|
214
|
+
| Schema only (`--schema-only`) | `MIGRATE_TO_D1_SCHEMA_ONLY`, `D1_MIGRATOR_SCHEMA_ONLY` |
|
|
215
|
+
| Interactive (`--interactive`) | `MIGRATE_TO_D1_INTERACTIVE`, `D1_MIGRATOR_INTERACTIVE` |
|
|
216
|
+
| Resume (`--resume`) | `MIGRATE_TO_D1_RESUME`, `D1_MIGRATOR_RESUME` |
|
|
217
|
+
| Migration ID (`--migration-id`) | `MIGRATE_TO_D1_MIGRATION_ID`, `D1_MIGRATOR_MIGRATION_ID` |
|
|
218
|
+
|
|
219
|
+
If `--source-connection` is not provided, the command automatically composes a URI from `DB_*` values for MySQL/PostgreSQL/SQL Server, and uses `DB_PATH`/`DB_DATABASE` for SQLite. Host fallback prefers `DB_READ_HOSTS`, then `DB_HOSTS`, then `DB_HOST`.
|
|
220
|
+
|
|
221
|
+
### Programmatic Usage
|
|
222
|
+
|
|
223
|
+
#### Basic Migration
|
|
224
|
+
|
|
225
|
+
```typescript
|
|
226
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
227
|
+
|
|
228
|
+
const config = {
|
|
229
|
+
sourceConnection: 'mysql://user:password@localhost:3306/mydb',
|
|
230
|
+
sourceDriver: 'mysql' as const,
|
|
231
|
+
targetDatabase: 'my-d1-db',
|
|
232
|
+
targetType: 'd1' as const,
|
|
233
|
+
batchSize: 1000,
|
|
234
|
+
checkpointInterval: 10000,
|
|
235
|
+
migrationId: 'migration-' + Date.now(),
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
try {
|
|
239
|
+
const progress = await D1Migrator.DataMigrator.migrateData(config);
|
|
240
|
+
|
|
241
|
+
console.log('Migration Results:');
|
|
242
|
+
console.log(`- Status: ${progress.status}`);
|
|
243
|
+
console.log(`- Tables: ${progress.totalTables}`);
|
|
244
|
+
console.log(`- Rows: ${progress.processedRows}/${progress.totalRows}`);
|
|
245
|
+
console.log(`- Errors: ${Object.keys(progress.errors).length}`);
|
|
246
|
+
|
|
247
|
+
if (progress.status === 'failed') {
|
|
248
|
+
console.error('Migration errors:', progress.errors);
|
|
249
|
+
}
|
|
250
|
+
} catch (error) {
|
|
251
|
+
console.error('Migration failed:', error);
|
|
252
|
+
}
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
#### Schema Analysis Only
|
|
256
|
+
|
|
257
|
+
```typescript
|
|
258
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
259
|
+
|
|
260
|
+
const connection = {
|
|
261
|
+
driver: 'mysql' as const,
|
|
262
|
+
connectionString: 'mysql://user:password@localhost:3306/mydb',
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
const schema = await D1Migrator.SchemaAnalyzer.analyzeSchema(connection);
|
|
266
|
+
|
|
267
|
+
console.log(`Found ${schema.tables.length} tables`);
|
|
268
|
+
schema.tables.forEach((table) => {
|
|
269
|
+
console.log(`- ${table.name}: ${table.columns.length} columns, ${table.rowCount} rows`);
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
// Check D1 compatibility
|
|
273
|
+
const compatibility = D1Migrator.SchemaAnalyzer.checkD1Compatibility(schema);
|
|
274
|
+
if (!compatibility.compatible) {
|
|
275
|
+
console.warn('Compatibility issues:', compatibility.issues);
|
|
276
|
+
}
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
#### Schema Conversion
|
|
280
|
+
|
|
281
|
+
```typescript
|
|
282
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
283
|
+
|
|
284
|
+
const sourceSchema = await D1Migrator.SchemaAnalyzer.analyzeSchema(connection);
|
|
285
|
+
const d1Schema = D1Migrator.SchemaBuilder.buildD1Schema(sourceSchema.tables, 'mysql');
|
|
286
|
+
|
|
287
|
+
// d1Schema contains D1-compatible CREATE TABLE statements
|
|
288
|
+
console.log(d1Schema);
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
#### Data Validation
|
|
292
|
+
|
|
293
|
+
```typescript
|
|
294
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
295
|
+
|
|
296
|
+
const results = await D1Migrator.DataValidator.validateMigration(
|
|
297
|
+
config,
|
|
298
|
+
sourceSchema,
|
|
299
|
+
targetDatabase
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
results.forEach((result) => {
|
|
303
|
+
console.log(`Table: ${result.table}`);
|
|
304
|
+
console.log(`- Source rows: ${result.sourceCount}`);
|
|
305
|
+
console.log(`- Target rows: ${result.targetCount}`);
|
|
306
|
+
console.log(`- Match: ${result.checksumMatch ? '✓' : '✗'}`);
|
|
307
|
+
|
|
308
|
+
if (!result.checksumMatch) {
|
|
309
|
+
console.warn('- Missing rows:', result.missingRows?.length);
|
|
310
|
+
console.warn('- Extra rows:', result.extraRows?.length);
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
## Connection Strings
|
|
316
|
+
|
|
317
|
+
### MySQL
|
|
318
|
+
|
|
319
|
+
```
|
|
320
|
+
mysql://[username]:[password]@[host]:[port]/[database]
|
|
321
|
+
|
|
322
|
+
Examples:
|
|
323
|
+
mysql://root:password@localhost:3306/mydb
|
|
324
|
+
mysql://user:pass@db.example.com:3306/production
|
|
325
|
+
mysql://root@127.0.0.1/app_db
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
### PostgreSQL
|
|
329
|
+
|
|
330
|
+
```
|
|
331
|
+
postgresql://[username]:[password]@[host]:[port]/[database]
|
|
332
|
+
|
|
333
|
+
Examples:
|
|
334
|
+
postgresql://user:password@localhost:5432/mydb
|
|
335
|
+
postgresql://postgres:secret@db.example.com:5432/prod
|
|
336
|
+
postgresql://user@127.0.0.1/app_db
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
### SQLite
|
|
340
|
+
|
|
341
|
+
```
|
|
342
|
+
sqlite://[path/to/database.db]
|
|
343
|
+
or
|
|
344
|
+
/path/to/database.db
|
|
345
|
+
|
|
346
|
+
Examples:
|
|
347
|
+
sqlite:///data/app.db
|
|
348
|
+
/Users/user/projects/db.sqlite
|
|
349
|
+
./data/local.db
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
### SQL Server
|
|
353
|
+
|
|
354
|
+
```
|
|
355
|
+
mssql://[username]:[password]@[host]:[port]/[database]
|
|
356
|
+
|
|
357
|
+
Examples:
|
|
358
|
+
mssql://sa:Password123@localhost:1433/mydb
|
|
359
|
+
mssql://user:pass@db.example.com:1433/production
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
## Configuration Reference
|
|
363
|
+
|
|
364
|
+
### MigrationConfig
|
|
365
|
+
|
|
366
|
+
```typescript
|
|
367
|
+
interface MigrationConfig {
|
|
368
|
+
// Source database connection
|
|
369
|
+
sourceConnection: string; // Connection URI
|
|
370
|
+
sourceDriver: SourceDatabaseDriver; // mysql | postgresql | sqlite | sqlserver
|
|
371
|
+
|
|
372
|
+
// Target D1 database
|
|
373
|
+
targetDatabase: string; // D1 database identifier
|
|
374
|
+
targetType: 'd1' | 'd1-remote'; // Local or remote D1
|
|
375
|
+
|
|
376
|
+
// Migration behavior (optional)
|
|
377
|
+
batchSize?: number; // Records per batch (default: 1000)
|
|
378
|
+
checkpointInterval?: number; // Save checkpoint every N rows (default: 10000)
|
|
379
|
+
dryRun?: boolean; // Test without changes (default: false)
|
|
380
|
+
interactive?: boolean; // Interactive mode (default: false)
|
|
381
|
+
migrationId?: string; // Migration identifier for resume
|
|
382
|
+
}
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
### MigrationProgress
|
|
386
|
+
|
|
387
|
+
```typescript
|
|
388
|
+
interface MigrationProgress {
|
|
389
|
+
migrationId: string;
|
|
390
|
+
currentTable: string;
|
|
391
|
+
status: 'pending' | 'processing' | 'completed' | 'failed';
|
|
392
|
+
processedRows: number;
|
|
393
|
+
totalRows: number;
|
|
394
|
+
totalTables: number;
|
|
395
|
+
percentage: number;
|
|
396
|
+
errors: Record<string, string>; // table -> error message
|
|
397
|
+
startTime?: Date;
|
|
398
|
+
endTime?: Date;
|
|
399
|
+
}
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
## Advanced Features
|
|
403
|
+
|
|
404
|
+
### Checkpoint Recovery
|
|
405
|
+
|
|
406
|
+
Migrations are automatically checkpointed every N rows (default 10,000). If a migration fails, you can resume from the last checkpoint:
|
|
407
|
+
|
|
408
|
+
```bash
|
|
409
|
+
# View checkpoint information
|
|
410
|
+
ls -la .wrangler/state/v3/migrations/
|
|
411
|
+
|
|
412
|
+
# Resume migration from checkpoint
|
|
413
|
+
zin migrate-to-d1 --resume --migration-id abc123def456
|
|
414
|
+
```
|
|
415
|
+
|
|
416
|
+
### Custom Batch Sizing
|
|
417
|
+
|
|
418
|
+
Batch size affects both performance and memory usage:
|
|
419
|
+
|
|
420
|
+
```typescript
|
|
421
|
+
// Small batches: slower but more memory-efficient
|
|
422
|
+
// Good for resource-constrained environments
|
|
423
|
+
batchSize: 500,
|
|
424
|
+
checkpointInterval: 2500,
|
|
425
|
+
|
|
426
|
+
// Large batches: faster but uses more memory
|
|
427
|
+
// Good for high-performance environments
|
|
428
|
+
batchSize: 5000,
|
|
429
|
+
checkpointInterval: 25000,
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
### Dry-Run Mode
|
|
433
|
+
|
|
434
|
+
Always test migrations in dry-run mode first to catch issues:
|
|
435
|
+
|
|
436
|
+
```bash
|
|
437
|
+
zin migrate-to-d1 \
|
|
438
|
+
--from mysql \
|
|
439
|
+
--source-connection "mysql://user:password@localhost:3306/mydb" \
|
|
440
|
+
--to d1 \
|
|
441
|
+
--target-database test-db \
|
|
442
|
+
--dry-run
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
### Interactive Mode
|
|
446
|
+
|
|
447
|
+
For complex migrations with compatibility warnings, use interactive mode:
|
|
448
|
+
|
|
449
|
+
```bash
|
|
450
|
+
zin migrate-to-d1 \
|
|
451
|
+
--from sqlserver \
|
|
452
|
+
--source-connection "mssql://user:password@localhost:1433/mydb" \
|
|
453
|
+
--to d1 \
|
|
454
|
+
--target-database target-db \
|
|
455
|
+
--interactive
|
|
456
|
+
```
|
|
457
|
+
|
|
458
|
+
The interactive mode will:
|
|
459
|
+
|
|
460
|
+
- Show all compatibility warnings
|
|
461
|
+
- Ask for confirmation before proceeding
|
|
462
|
+
- Suggest workarounds for unsupported features
|
|
463
|
+
- Allow manual schema adjustments
|
|
464
|
+
|
|
465
|
+
## Type Conversions
|
|
466
|
+
|
|
467
|
+
### Automatic Data Transformations
|
|
468
|
+
|
|
469
|
+
The migrator automatically converts data types for D1 compatibility:
|
|
470
|
+
|
|
471
|
+
| Source Type | SQLite Type | Notes |
|
|
472
|
+
| ------------------- | --------------- | -------------------------------- |
|
|
473
|
+
| DATETIME, TIMESTAMP | TEXT (ISO 8601) | Converted to ISO 8601 strings |
|
|
474
|
+
| BIGINT | TEXT | Large integers stored as strings |
|
|
475
|
+
| DECIMAL, NUMERIC | TEXT | Precision preserved as strings |
|
|
476
|
+
| JSON | TEXT | JSON objects stored as strings |
|
|
477
|
+
| BLOB | BLOB | Binary data preserved |
|
|
478
|
+
| NULL | NULL | Null values preserved |
|
|
479
|
+
|
|
480
|
+
### Manual Value Transformation
|
|
481
|
+
|
|
482
|
+
For custom value transformations:
|
|
483
|
+
|
|
484
|
+
```typescript
|
|
485
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
486
|
+
|
|
487
|
+
const transformed = D1Migrator.TypeConverter.transformValue(value, sourceType, 'sqlite');
|
|
488
|
+
```
|
|
489
|
+
|
|
490
|
+
## Error Handling
|
|
491
|
+
|
|
492
|
+
### Common Errors and Solutions
|
|
493
|
+
|
|
494
|
+
#### Connection Failed
|
|
495
|
+
|
|
496
|
+
```
|
|
497
|
+
Error: Unable to connect to source database
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
**Solution**: Check connection string format and network connectivity.
|
|
501
|
+
|
|
502
|
+
```bash
|
|
503
|
+
# Verify connection
|
|
504
|
+
mysql -h localhost -u user -p -D database -e "SELECT 1;"
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
#### Schema Incompatibility
|
|
508
|
+
|
|
509
|
+
```
|
|
510
|
+
Error: Schema compatibility issues prevent migration
|
|
511
|
+
- Unsupported column type: GEOMETRY
|
|
512
|
+
- Unsupported feature: PARTITION BY
|
|
513
|
+
```
|
|
514
|
+
|
|
515
|
+
**Solution**: Use interactive mode to review and accept changes:
|
|
516
|
+
|
|
517
|
+
```bash
|
|
518
|
+
zin migrate-to-d1 --from mysql --to d1 --interactive
|
|
519
|
+
```
|
|
520
|
+
|
|
521
|
+
#### Row Count Mismatch
|
|
522
|
+
|
|
523
|
+
```
|
|
524
|
+
Error: Data migration verification failed
|
|
525
|
+
Expected rows: 1000, Inserted rows: 998
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
**Solution**: Review specific table logs:
|
|
529
|
+
|
|
530
|
+
1. Check for NULL values in unique/primary key columns
|
|
531
|
+
2. Verify foreign key constraints on source
|
|
532
|
+
3. Run validation to identify missing rows:
|
|
533
|
+
|
|
534
|
+
```typescript
|
|
535
|
+
const validation = await D1Migrator.DataValidator.validateMigration(config);
|
|
536
|
+
validation.forEach((result) => {
|
|
537
|
+
if (!result.checksumMatch) {
|
|
538
|
+
console.log(`Missing rows in ${result.table}:`, result.missingRows);
|
|
539
|
+
}
|
|
540
|
+
});
|
|
541
|
+
```
|
|
542
|
+
|
|
543
|
+
#### Out of Memory
|
|
544
|
+
|
|
545
|
+
```
|
|
546
|
+
Error: FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
**Solution**: Reduce batch size:
|
|
550
|
+
|
|
551
|
+
```bash
|
|
552
|
+
zin migrate-to-d1 \
|
|
553
|
+
--source-connection "..." \
|
|
554
|
+
--batch-size 500 \
|
|
555
|
+
--checkpoint-interval 2500
|
|
556
|
+
```
|
|
557
|
+
|
|
558
|
+
## Performance Tuning
|
|
559
|
+
|
|
560
|
+
### Optimization Guidelines
|
|
561
|
+
|
|
562
|
+
1. **Batch Size**: Balance between memory and speed
|
|
563
|
+
- Test with 1000-2000 records first
|
|
564
|
+
- Increase if memory available and no OOM errors
|
|
565
|
+
- Decrease if experiencing memory pressure
|
|
566
|
+
|
|
567
|
+
2. **Checkpoint Interval**: Balance between durability and speed
|
|
568
|
+
- Set to 5-10x batch size
|
|
569
|
+
- More checkpoints = slower but safer
|
|
570
|
+
- Fewer checkpoints = faster but riskier
|
|
571
|
+
|
|
572
|
+
3. **Connection Pooling**: Configure at adapter level
|
|
573
|
+
- MySQL: Best with 5-10 connections
|
|
574
|
+
- PostgreSQL: Best with 2-5 connections
|
|
575
|
+
- SQLite: Single connection optimal
|
|
576
|
+
|
|
577
|
+
4. **Network**: For remote sources
|
|
578
|
+
- Ensure low latency connection
|
|
579
|
+
- Consider regional endpoints if available
|
|
580
|
+
- Use compression if supported
|
|
581
|
+
|
|
582
|
+
### Benchmarks (Typical Performance)
|
|
583
|
+
|
|
584
|
+
| Source | Size | Time | Type |
|
|
585
|
+
| --------------- | --------- | ------- | ---------- |
|
|
586
|
+
| MySQL 5.7 | 100k rows | ~5 min | 1000 batch |
|
|
587
|
+
| PostgreSQL 13 | 500k rows | ~25 min | 2000 batch |
|
|
588
|
+
| SQL Server 2019 | 1M rows | ~50 min | 2000 batch |
|
|
589
|
+
| SQLite 3 | 100k rows | ~2 min | 1000 batch |
|
|
590
|
+
|
|
591
|
+
## Examples
|
|
592
|
+
|
|
593
|
+
### Complete Migration Workflow
|
|
594
|
+
|
|
595
|
+
```typescript
|
|
596
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
597
|
+
import { Logger } from '@zintrust/core';
|
|
598
|
+
|
|
599
|
+
async function migrateDatabase() {
|
|
600
|
+
try {
|
|
601
|
+
// Step 1: Analyze source schema
|
|
602
|
+
Logger.info('Analyzing source database...');
|
|
603
|
+
const connection = {
|
|
604
|
+
driver: 'mysql' as const,
|
|
605
|
+
connectionString: process.env.DB_SOURCE_URL!,
|
|
606
|
+
};
|
|
607
|
+
|
|
608
|
+
const schema = await D1Migrator.SchemaAnalyzer.analyzeSchema(connection);
|
|
609
|
+
Logger.info(`Found ${schema.tables.length} tables`);
|
|
610
|
+
|
|
611
|
+
// Step 2: Check D1 compatibility
|
|
612
|
+
const compatibility = D1Migrator.SchemaAnalyzer.checkD1Compatibility(schema);
|
|
613
|
+
if (!compatibility.compatible) {
|
|
614
|
+
throw new Error(`Compatibility issues: ${compatibility.issues.join(', ')}`);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// Step 3: Build D1 schema
|
|
618
|
+
const d1Schema = D1Migrator.SchemaBuilder.buildD1Schema(schema.tables, 'mysql');
|
|
619
|
+
Logger.info('D1 schema built successfully');
|
|
620
|
+
|
|
621
|
+
// Step 4: Migrate data
|
|
622
|
+
Logger.info('Starting data migration...');
|
|
623
|
+
const config = {
|
|
624
|
+
sourceConnection: process.env.DB_SOURCE_URL!,
|
|
625
|
+
sourceDriver: 'mysql' as const,
|
|
626
|
+
targetDatabase: process.env.D1_DATABASE!,
|
|
627
|
+
targetType: 'd1' as const,
|
|
628
|
+
batchSize: 1000,
|
|
629
|
+
checkpointInterval: 10000,
|
|
630
|
+
migrationId: 'migration-' + Date.now(),
|
|
631
|
+
};
|
|
632
|
+
|
|
633
|
+
const progress = await D1Migrator.DataMigrator.migrateData(config);
|
|
634
|
+
|
|
635
|
+
if (progress.status === 'failed') {
|
|
636
|
+
Logger.error('Migration failed:', progress.errors);
|
|
637
|
+
throw new Error('Migration failed with errors');
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
// Step 5: Validate migration
|
|
641
|
+
Logger.info('Validating migrated data...');
|
|
642
|
+
const validation = await D1Migrator.DataValidator.validateMigration(
|
|
643
|
+
config,
|
|
644
|
+
schema,
|
|
645
|
+
process.env.D1_DATABASE!
|
|
646
|
+
);
|
|
647
|
+
|
|
648
|
+
const allValid = validation.every((r) => r.checksumMatch);
|
|
649
|
+
if (!allValid) {
|
|
650
|
+
Logger.warn('Validation warnings found');
|
|
651
|
+
validation.forEach((r) => {
|
|
652
|
+
if (!r.checksumMatch) {
|
|
653
|
+
Logger.warn(`${r.table}: source=${r.sourceCount}, target=${r.targetCount}`);
|
|
654
|
+
}
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
Logger.info('Migration completed successfully!');
|
|
659
|
+
Logger.info(`Total rows: ${progress.processedRows}`);
|
|
660
|
+
Logger.info(
|
|
661
|
+
`Duration: ${(progress.endTime!.getTime() - progress.startTime!.getTime()) / 1000}s`
|
|
662
|
+
);
|
|
663
|
+
} catch (error) {
|
|
664
|
+
Logger.error('Migration failed:', error);
|
|
665
|
+
throw error;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// Execute
|
|
670
|
+
migrateDatabase().catch(console.error);
|
|
671
|
+
```
|
|
672
|
+
|
|
673
|
+
### Monitor Migration Progress
|
|
674
|
+
|
|
675
|
+
```typescript
|
|
676
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
677
|
+
|
|
678
|
+
async function monitorMigration(config: MigrationConfig) {
|
|
679
|
+
const trackProgress = setInterval(async () => {
|
|
680
|
+
try {
|
|
681
|
+
const state = await D1Migrator.CheckpointManager.getCheckpointState(config.migrationId);
|
|
682
|
+
|
|
683
|
+
if (state) {
|
|
684
|
+
const percentage = (state.processedRows / state.totalRows) * 100;
|
|
685
|
+
console.log(
|
|
686
|
+
`Progress: ${percentage.toFixed(1)}% (${state.processedRows}/${state.totalRows})`
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
} catch (error) {
|
|
690
|
+
console.error('Failed to get progress:', error);
|
|
691
|
+
}
|
|
692
|
+
}, 5000); // Update every 5 seconds
|
|
693
|
+
|
|
694
|
+
const progress = await D1Migrator.DataMigrator.migrateData(config);
|
|
695
|
+
|
|
696
|
+
clearInterval(trackProgress);
|
|
697
|
+
return progress;
|
|
698
|
+
}
|
|
699
|
+
```
|
|
700
|
+
|
|
701
|
+
## Troubleshooting
|
|
702
|
+
|
|
703
|
+
### Debug Logging
|
|
704
|
+
|
|
705
|
+
Enable verbose logging to diagnose issues:
|
|
706
|
+
|
|
707
|
+
```bash
|
|
708
|
+
LOG_LEVEL=debug zin migrate-to-d1 \
|
|
709
|
+
--from mysql \
|
|
710
|
+
--source-connection "mysql://user:password@localhost:3306/mydb" \
|
|
711
|
+
--to d1 \
|
|
712
|
+
--target-database target-db
|
|
713
|
+
```
|
|
714
|
+
|
|
715
|
+
### Test Connection
|
|
716
|
+
|
|
717
|
+
Verify source database connectivity:
|
|
718
|
+
|
|
719
|
+
```bash
|
|
720
|
+
# MySQL
|
|
721
|
+
mysql -h localhost -u user -p -D database -e "SELECT 1;"
|
|
722
|
+
|
|
723
|
+
# PostgreSQL
|
|
724
|
+
psql -h localhost -U user -d database -c "SELECT 1;"
|
|
725
|
+
|
|
726
|
+
# SQL Server
|
|
727
|
+
sqlcmd -S localhost -U sa -P password -Q "SELECT 1;"
|
|
728
|
+
|
|
729
|
+
# SQLite
|
|
730
|
+
sqlite3 /path/to/database.db "SELECT 1;"
|
|
731
|
+
```
|
|
732
|
+
|
|
733
|
+
### Inspect D1 Database
|
|
734
|
+
|
|
735
|
+
```bash
|
|
736
|
+
# List D1 databases
|
|
737
|
+
wrangler d1 list
|
|
738
|
+
|
|
739
|
+
# Query D1 database
|
|
740
|
+
wrangler d1 execute my-d1-db --remote --command "SELECT COUNT(*) FROM table_name;"
|
|
741
|
+
|
|
742
|
+
# Backup D1
|
|
743
|
+
wrangler d1 backup create my-d1-db --remote
|
|
744
|
+
```
|
|
745
|
+
|
|
746
|
+
### Review Checkpoint Data
|
|
747
|
+
|
|
748
|
+
```bash
|
|
749
|
+
# Find checkpoint files
|
|
750
|
+
find .wrangler/state/v3/migrations -name "*.json" -type f
|
|
751
|
+
|
|
752
|
+
# View checkpoint content
|
|
753
|
+
cat .wrangler/state/v3/migrations/migration-123456.json
|
|
754
|
+
```
|
|
755
|
+
|
|
756
|
+
## Architecture
|
|
757
|
+
|
|
758
|
+
### Module Structure
|
|
759
|
+
|
|
760
|
+
```
|
|
761
|
+
packages/d1-migrator/
|
|
762
|
+
├── src/
|
|
763
|
+
│ ├── index.ts # Entry point, sealed namespace export
|
|
764
|
+
│ ├── types.ts # Type definitions
|
|
765
|
+
│ ├── cli/ # CLI components
|
|
766
|
+
│ │ ├── MigrateToD1Command.ts # CLI command definition
|
|
767
|
+
│ │ ├── DataMigrator.ts # Core migration orchestrator
|
|
768
|
+
│ │ ├── SchemaAnalyzer.ts # Source schema introspection
|
|
769
|
+
│ │ └── ProgressTracker.ts # Migration progress tracking
|
|
770
|
+
│ ├── schema/ # Schema conversion
|
|
771
|
+
│ │ ├── SchemaBuilder.ts # Builds D1-compatible schemas
|
|
772
|
+
│ │ ├── TypeConverter.ts # Type transformations
|
|
773
|
+
│ │ └── Validator.ts # Schema validation
|
|
774
|
+
│ └── utils/ # Utilities
|
|
775
|
+
│ ├── CheckpointManager.ts # Resumable migration checkpoints
|
|
776
|
+
│ └── DataValidator.ts # Data integrity validation
|
|
777
|
+
└── package.json
|
|
778
|
+
```
|
|
779
|
+
|
|
780
|
+
### Data Flow
|
|
781
|
+
|
|
782
|
+
```
|
|
783
|
+
Source Database
|
|
784
|
+
↓
|
|
785
|
+
[SchemaAnalyzer] ← Introspect tables, columns, keys, indexes
|
|
786
|
+
↓
|
|
787
|
+
Database Schema Object
|
|
788
|
+
↓
|
|
789
|
+
[Compatibility Check] ← Verify D1 support
|
|
790
|
+
↓
|
|
791
|
+
[SchemaBuilder] ← Convert to D1-compatible schema
|
|
792
|
+
↓
|
|
793
|
+
[DataMigrator] ← Migrate data in batches
|
|
794
|
+
├─ [TypeConverter] ← Transform values
|
|
795
|
+
├─ [CheckpointManager] ← Save progress
|
|
796
|
+
└─ [DataValidator] ← Verify rows
|
|
797
|
+
↓
|
|
798
|
+
D1 Database
|
|
799
|
+
```
|
|
800
|
+
|
|
801
|
+
## Development
|
|
802
|
+
|
|
803
|
+
### Building from Source
|
|
804
|
+
|
|
805
|
+
```bash
|
|
806
|
+
# Install dependencies
|
|
807
|
+
npm install
|
|
808
|
+
|
|
809
|
+
# Build TypeScript
|
|
810
|
+
npm run build
|
|
811
|
+
|
|
812
|
+
# Run tests
|
|
813
|
+
npm test
|
|
814
|
+
|
|
815
|
+
# Type checking
|
|
816
|
+
npm run type-check
|
|
817
|
+
|
|
818
|
+
# Linting
|
|
819
|
+
npm run lint
|
|
820
|
+
```
|
|
821
|
+
|
|
822
|
+
### Testing Locally
|
|
823
|
+
|
|
824
|
+
```typescript
|
|
825
|
+
// tests/integration/migration.test.ts
|
|
826
|
+
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
|
827
|
+
import { D1Migrator } from '@zintrust/d1-migrator';
|
|
828
|
+
|
|
829
|
+
describe('D1 Migration', () => {
|
|
830
|
+
it('should migrate MySQL data successfully', async () => {
|
|
831
|
+
const config = {
|
|
832
|
+
sourceConnection: process.env.TEST_MYSQL_URL!,
|
|
833
|
+
sourceDriver: 'mysql' as const,
|
|
834
|
+
targetDatabase: 'test-d1',
|
|
835
|
+
targetType: 'd1' as const,
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
const progress = await D1Migrator.DataMigrator.migrateData(config);
|
|
839
|
+
expect(progress.status).toBe('completed');
|
|
840
|
+
expect(progress.processedRows).toBeGreaterThan(0);
|
|
841
|
+
});
|
|
842
|
+
});
|
|
843
|
+
```
|
|
844
|
+
|
|
845
|
+
## Contributing
|
|
846
|
+
|
|
847
|
+
Contributions are welcome! Please read [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on our code of conduct and process for submitting pull requests.
|
|
848
|
+
|
|
849
|
+
### Bug Reports
|
|
850
|
+
|
|
851
|
+
[GitHub Issues](https://github.com/ZinTrust/zintrust/issues)
|
|
852
|
+
|
|
853
|
+
## License
|
|
854
|
+
|
|
855
|
+
MIT License - see [LICENSE.md](../../LICENSE.md) for details
|
|
856
|
+
|
|
857
|
+
## Support
|
|
858
|
+
|
|
859
|
+
- **Documentation**: [Full Documentation](../../docs/adapters.md)
|
|
860
|
+
- **Issues**: [GitHub Issues](https://github.com/ZinTrust/zintrust/issues)
|
|
861
|
+
- **Discussions**: [GitHub Discussions](https://github.com/ZinTrust/zintrust/discussions)
|
|
862
|
+
- **Email**: support@zintrust.dev
|
|
863
|
+
|
|
864
|
+
## Roadmap
|
|
865
|
+
|
|
866
|
+
- [ ] Resume from checkpoints (in progress)
|
|
867
|
+
- [ ] MongoDB source support
|
|
868
|
+
- [ ] GraphQL schema introspection
|
|
869
|
+
- [ ] Data anonymization during migration
|
|
870
|
+
- [ ] Real-time replication mode
|
|
871
|
+
- [ ] Web UI for migration management
|
|
872
|
+
|
|
873
|
+
## Related Packages
|
|
874
|
+
|
|
875
|
+
- [@zintrust/core](https://www.npmjs.com/package/@zintrust/core) - Core framework
|
|
876
|
+
- [@zintrust/db-mysql](https://www.npmjs.com/package/@zintrust/db-mysql) - MySQL adapter
|
|
877
|
+
- [@zintrust/db-postgres](https://www.npmjs.com/package/@zintrust/db-postgres) - PostgreSQL adapter
|
|
878
|
+
- [@zintrust/db-sqlite](https://www.npmjs.com/package/@zintrust/db-sqlite) - SQLite adapter
|
|
879
|
+
- [@zintrust/db-sqlserver](https://www.npmjs.com/package/@zintrust/db-sqlserver) - SQL Server adapter
|
|
880
|
+
- [@zintrust/db-d1](https://www.npmjs.com/package/@zintrust/db-d1) - D1 adapter
|
|
881
|
+
|
|
882
|
+
---
|
|
883
|
+
|
|
884
|
+
Made with ❤️ by ZinTrust
|
package/dist/build-manifest.json
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/d1-migrator",
|
|
3
3
|
"version": "0.1.0",
|
|
4
|
-
"buildDate": "2026-03-
|
|
4
|
+
"buildDate": "2026-03-20T13:11:40.471Z",
|
|
5
5
|
"buildEnvironment": {
|
|
6
6
|
"node": "v22.22.1",
|
|
7
7
|
"platform": "darwin",
|
|
8
8
|
"arch": "arm64"
|
|
9
9
|
},
|
|
10
10
|
"git": {
|
|
11
|
-
"commit": "
|
|
12
|
-
"branch": "
|
|
11
|
+
"commit": "e0e7e31c",
|
|
12
|
+
"branch": "release"
|
|
13
13
|
},
|
|
14
14
|
"package": {
|
|
15
15
|
"engines": {
|
|
@@ -28,8 +28,8 @@
|
|
|
28
28
|
},
|
|
29
29
|
"files": {
|
|
30
30
|
"build-manifest.json": {
|
|
31
|
-
"size":
|
|
32
|
-
"sha256": "
|
|
31
|
+
"size": 5804,
|
|
32
|
+
"sha256": "d093f50bf4b90acf6721932a0c2a3eb6a27a3790a38947e80b653ea5a3e67726"
|
|
33
33
|
},
|
|
34
34
|
"cli/DataMigrator.d.ts": {
|
|
35
35
|
"size": 3355,
|
|
@@ -89,15 +89,15 @@
|
|
|
89
89
|
},
|
|
90
90
|
"index.js": {
|
|
91
91
|
"size": 1204,
|
|
92
|
-
"sha256": "
|
|
92
|
+
"sha256": "a4c5cc454b6a6ebd61ec92822d3147b75a39c588f518bb2bb4c9c6f824d73bf9"
|
|
93
93
|
},
|
|
94
94
|
"register.d.ts": {
|
|
95
|
-
"size":
|
|
96
|
-
"sha256": "
|
|
95
|
+
"size": 159,
|
|
96
|
+
"sha256": "e60a21d17e51095a4a8a9c4a2fae45c902305635998782b8194eb968d8083cb2"
|
|
97
97
|
},
|
|
98
98
|
"register.d.ts.map": {
|
|
99
|
-
"size":
|
|
100
|
-
"sha256": "
|
|
99
|
+
"size": 211,
|
|
100
|
+
"sha256": "6743ba50c90440882096f5eaf09a2fff4c901ba238c604d2240ba072b5754632"
|
|
101
101
|
},
|
|
102
102
|
"register.js": {
|
|
103
103
|
"size": 857,
|
|
@@ -140,12 +140,12 @@
|
|
|
140
140
|
"sha256": "e8349ddbbf3c969468d741e634a50422847052f21afa38e63493853273945e6e"
|
|
141
141
|
},
|
|
142
142
|
"types.d.ts": {
|
|
143
|
-
"size":
|
|
144
|
-
"sha256": "
|
|
143
|
+
"size": 4098,
|
|
144
|
+
"sha256": "a03fc55513b8d5c7d0cae3f417fad9e5dcaaee511b4868cdae8cd3061af84a6c"
|
|
145
145
|
},
|
|
146
146
|
"types.d.ts.map": {
|
|
147
|
-
"size":
|
|
148
|
-
"sha256": "
|
|
147
|
+
"size": 4181,
|
|
148
|
+
"sha256": "4dff98570f92baca6dee52a13a1942e124e3b6a386b0d19381b8e7319a623bae"
|
|
149
149
|
},
|
|
150
150
|
"types.js": {
|
|
151
151
|
"size": 85,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAiEF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoEzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAWH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAiEF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAoEzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAoChB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAiBpF;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAyEtD;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAkBrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,aACrB,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA4CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAkClB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
|
package/dist/cli/DataMigrator.js
CHANGED
|
@@ -3,13 +3,13 @@
|
|
|
3
3
|
* Data Migrator
|
|
4
4
|
* Handles the actual data migration between databases
|
|
5
5
|
*/
|
|
6
|
-
import { ErrorFactory, Logger } from '@zintrust/core';
|
|
6
|
+
import { ErrorFactory, LocalD1Resolver, Logger } from '@zintrust/core';
|
|
7
7
|
import { MySQLAdapter } from '@zintrust/db-mysql';
|
|
8
8
|
import { PostgreSQLAdapter } from '@zintrust/db-postgres';
|
|
9
9
|
import { SQLiteAdapter } from '@zintrust/db-sqlite';
|
|
10
10
|
import { SQLServerAdapter } from '@zintrust/db-sqlserver';
|
|
11
|
-
import { SchemaBuilder } from '../schema/SchemaBuilder';
|
|
12
|
-
import { SchemaAnalyzer } from './SchemaAnalyzer';
|
|
11
|
+
import { SchemaBuilder } from '../schema/SchemaBuilder.js';
|
|
12
|
+
import { SchemaAnalyzer } from './SchemaAnalyzer.js';
|
|
13
13
|
const parseConnectionDetails = (connectionString, defaultPort, defaultDatabase, defaultUsername) => {
|
|
14
14
|
try {
|
|
15
15
|
const parsed = new URL(connectionString);
|
|
@@ -192,14 +192,17 @@ export const DataMigrator = Object.freeze({
|
|
|
192
192
|
connected: true,
|
|
193
193
|
};
|
|
194
194
|
if (config.targetType === 'd1') {
|
|
195
|
-
const
|
|
195
|
+
const projectRoot = process.cwd();
|
|
196
|
+
const resolvedTarget = LocalD1Resolver.resolveD1Binding(projectRoot, config.targetDatabase);
|
|
197
|
+
const d1LocalPath = await LocalD1Resolver.resolveLocalD1SqlitePath(projectRoot, config.targetDatabase);
|
|
196
198
|
const d1Local = SQLiteAdapter.create({ driver: 'sqlite', database: d1LocalPath });
|
|
197
199
|
try {
|
|
198
200
|
await d1Local.connect();
|
|
199
201
|
connection.adapter = d1Local;
|
|
202
|
+
connection.database = resolvedTarget.databaseName;
|
|
200
203
|
}
|
|
201
204
|
catch (error) {
|
|
202
|
-
|
|
205
|
+
throw ErrorFactory.createConnectionError(`Unable to connect resolved local D1 path ${d1LocalPath}: ${String(error)}`);
|
|
203
206
|
}
|
|
204
207
|
}
|
|
205
208
|
Logger.info('✓ Target D1 database connected');
|
|
@@ -210,8 +213,7 @@ export const DataMigrator = Object.freeze({
|
|
|
210
213
|
*/
|
|
211
214
|
async prepareTargetSchema(sourceConnection, targetConnection, config) {
|
|
212
215
|
if (!targetConnection.adapter) {
|
|
213
|
-
|
|
214
|
-
return;
|
|
216
|
+
throw ErrorFactory.createConnectionError('No target adapter available for D1 schema preparation');
|
|
215
217
|
}
|
|
216
218
|
Logger.info('Preparing target D1 schema...');
|
|
217
219
|
const sourceSchema = await SchemaAnalyzer.analyzeSchema({
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { ErrorFactory, Logger } from '@zintrust/core';
|
|
6
6
|
import { BaseCommand } from '@zintrust/core/cli';
|
|
7
|
-
import { SchemaBuilder } from '../schema/SchemaBuilder';
|
|
8
|
-
import { SchemaValidator } from '../schema/Validator';
|
|
9
|
-
import { DataMigrator } from './DataMigrator';
|
|
10
|
-
import { SchemaAnalyzer } from './SchemaAnalyzer';
|
|
7
|
+
import { SchemaBuilder } from '../schema/SchemaBuilder.js';
|
|
8
|
+
import { SchemaValidator } from '../schema/Validator.js';
|
|
9
|
+
import { DataMigrator } from './DataMigrator.js';
|
|
10
|
+
import { SchemaAnalyzer } from './SchemaAnalyzer.js';
|
|
11
11
|
const SOURCE_DRIVER_MAP = Object.freeze({
|
|
12
12
|
mysql: 'mysql',
|
|
13
13
|
postgresql: 'postgresql',
|
package/dist/index.js
CHANGED
|
@@ -3,18 +3,18 @@
|
|
|
3
3
|
* Migrate any database to Cloudflare D1 with resumable operations
|
|
4
4
|
*/
|
|
5
5
|
// CLI Commands
|
|
6
|
-
import { MigrateToD1Command } from './cli/MigrateToD1Command';
|
|
6
|
+
import { MigrateToD1Command } from './cli/MigrateToD1Command.js';
|
|
7
7
|
// Utilities
|
|
8
|
-
import { CheckpointManager } from './utils/CheckpointManager';
|
|
9
|
-
import { DataValidator } from './utils/DataValidator';
|
|
8
|
+
import { CheckpointManager } from './utils/CheckpointManager.js';
|
|
9
|
+
import { DataValidator } from './utils/DataValidator.js';
|
|
10
10
|
// CLI Components
|
|
11
|
-
import { DataMigrator } from './cli/DataMigrator';
|
|
12
|
-
import { ProgressTracker } from './cli/ProgressTracker';
|
|
13
|
-
import { SchemaAnalyzer } from './cli/SchemaAnalyzer';
|
|
11
|
+
import { DataMigrator } from './cli/DataMigrator.js';
|
|
12
|
+
import { ProgressTracker } from './cli/ProgressTracker.js';
|
|
13
|
+
import { SchemaAnalyzer } from './cli/SchemaAnalyzer.js';
|
|
14
14
|
// Schema Components
|
|
15
|
-
import { SchemaBuilder } from './schema/SchemaBuilder';
|
|
16
|
-
import { TypeConverter } from './schema/TypeConverter';
|
|
17
|
-
import { SchemaValidator } from './schema/Validator';
|
|
15
|
+
import { SchemaBuilder } from './schema/SchemaBuilder.js';
|
|
16
|
+
import { TypeConverter } from './schema/TypeConverter.js';
|
|
17
|
+
import { SchemaValidator } from './schema/Validator.js';
|
|
18
18
|
export const D1Migrator = Object.freeze({
|
|
19
19
|
// CLI Commands
|
|
20
20
|
MigrateToD1Command,
|
package/dist/register.js
CHANGED
|
@@ -3,7 +3,7 @@ const commandModule = (await (async () => {
|
|
|
3
3
|
return (await import('./cli/MigrateToD1Command.js'));
|
|
4
4
|
}
|
|
5
5
|
catch {
|
|
6
|
-
return (await import('./cli/MigrateToD1Command'));
|
|
6
|
+
return (await import('./cli/MigrateToD1Command.js'));
|
|
7
7
|
}
|
|
8
8
|
})());
|
|
9
9
|
export function registerD1MigratorCommand(registry) {
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* Builds D1/SQLite compatible schemas from source schemas
|
|
4
4
|
*/
|
|
5
5
|
import { Logger } from '@zintrust/core';
|
|
6
|
-
import { DataValidator } from '../utils/DataValidator';
|
|
7
|
-
import { TypeConverter } from './TypeConverter';
|
|
6
|
+
import { DataValidator } from '../utils/DataValidator.js';
|
|
7
|
+
import { TypeConverter } from './TypeConverter.js';
|
|
8
8
|
/**
|
|
9
9
|
* SchemaBuilder - Sealed namespace for schema building
|
|
10
10
|
* Provides D1 schema generation from source schemas
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/d1-migrator",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.5",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -22,11 +22,22 @@
|
|
|
22
22
|
"node": ">=20.0.0"
|
|
23
23
|
},
|
|
24
24
|
"peerDependencies": {
|
|
25
|
-
"@zintrust/core": "^0.4.
|
|
25
|
+
"@zintrust/core": "^0.4.5"
|
|
26
26
|
},
|
|
27
27
|
"publishConfig": {
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
|
+
"keywords": [
|
|
31
|
+
"zintrust",
|
|
32
|
+
"cloudflare",
|
|
33
|
+
"d1",
|
|
34
|
+
"migration",
|
|
35
|
+
"database"
|
|
36
|
+
],
|
|
37
|
+
"scripts": {
|
|
38
|
+
"build": "tsc -p tsconfig.json && node ../../scripts/fix-dist-esm-imports.mjs dist",
|
|
39
|
+
"prepublishOnly": "npm run build"
|
|
40
|
+
},
|
|
30
41
|
"dependencies": {
|
|
31
42
|
"@zintrust/db-mysql": "0.4.0",
|
|
32
43
|
"@zintrust/db-postgres": "0.4.0",
|
|
@@ -34,4 +45,4 @@
|
|
|
34
45
|
"@zintrust/db-sqlserver": "0.4.0",
|
|
35
46
|
"@zintrust/db-d1": "0.4.0"
|
|
36
47
|
}
|
|
37
|
-
}
|
|
48
|
+
}
|