@zintrust/d1-migrator 2.0.3 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/build-manifest.json +179 -0
- package/dist/cli/DataMigrator.d.ts +5 -0
- package/dist/cli/DataMigrator.d.ts.map +1 -1
- package/dist/cli/DataMigrator.js +318 -19
- package/dist/cli/MigrateToD1Command.d.ts.map +1 -1
- package/dist/cli/MigrateToD1Command.js +3 -2
- package/dist/cli/ProgressTracker.js +1 -1
- package/dist/cli/SchemaAnalyzer.d.ts.map +1 -1
- package/dist/cli/SchemaAnalyzer.js +2 -1
- package/dist/schema/SchemaBuilder.d.ts.map +1 -1
- package/dist/schema/SchemaBuilder.js +2 -1
- package/dist/schema/TypeConverter.js +1 -1
- package/dist/utils/CheckpointManager.d.ts.map +1 -1
- package/dist/utils/CheckpointManager.js +3 -1
- package/dist/utils/DataValidator.js +1 -1
- package/package.json +7 -7
package/README.md
CHANGED
|
@@ -226,6 +226,9 @@ The command supports env-based execution for all CLI settings.
|
|
|
226
226
|
| Interactive (`--interactive`) | `MIGRATE_TO_D1_INTERACTIVE`, `D1_MIGRATOR_INTERACTIVE` |
|
|
227
227
|
| Resume (`--resume`) | `MIGRATE_TO_D1_RESUME`, `D1_MIGRATOR_RESUME` |
|
|
228
228
|
| Migration ID (`--migration-id`) | `MIGRATE_TO_D1_MIGRATION_ID`, `D1_MIGRATOR_MIGRATION_ID` |
|
|
229
|
+
| Group Small Tables | `MIGRATE_TO_D1_GROUP_SMALL_TABLES` (Default: `true`) |
|
|
230
|
+
| Max Group Rows | `MIGRATE_TO_D1_MAX_GROUP_ROWS` (Default: `50000`) |
|
|
231
|
+
| Max Group Size (MB) | `MIGRATE_TO_D1_MAX_GROUP_SIZE_MB` (Default: `10`) |
|
|
229
232
|
|
|
230
233
|
If `--source-connection` is not provided, the command automatically composes a URI from `DB_*` values for MySQL/PostgreSQL/SQL Server, and uses `DB_PATH`/`DB_DATABASE` for SQLite. Host fallback prefers `DB_READ_HOSTS`, then `DB_HOSTS`, then `DB_HOST`.
|
|
231
234
|
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@zintrust/d1-migrator",
|
|
3
|
+
"version": "2.1.3",
|
|
4
|
+
"buildDate": "2026-05-27T04:01:34.093Z",
|
|
5
|
+
"buildEnvironment": {
|
|
6
|
+
"node": "v22.22.1",
|
|
7
|
+
"platform": "darwin",
|
|
8
|
+
"arch": "arm64"
|
|
9
|
+
},
|
|
10
|
+
"git": {
|
|
11
|
+
"commit": "c82f6263",
|
|
12
|
+
"branch": "release"
|
|
13
|
+
},
|
|
14
|
+
"package": {
|
|
15
|
+
"engines": {
|
|
16
|
+
"node": ">=20.0.0"
|
|
17
|
+
},
|
|
18
|
+
"dependencies": [
|
|
19
|
+
"@zintrust/db-d1",
|
|
20
|
+
"@zintrust/db-mysql",
|
|
21
|
+
"@zintrust/db-postgres",
|
|
22
|
+
"@zintrust/db-sqlite",
|
|
23
|
+
"@zintrust/db-sqlserver"
|
|
24
|
+
],
|
|
25
|
+
"peerDependencies": [
|
|
26
|
+
"@zintrust/core"
|
|
27
|
+
]
|
|
28
|
+
},
|
|
29
|
+
"files": {
|
|
30
|
+
"build-manifest.json": {
|
|
31
|
+
"size": 5808,
|
|
32
|
+
"sha256": "bd95f5fae475bd01f7504eb6a8f961e00491ccb8797aab36134f7d88fd9f2c13"
|
|
33
|
+
},
|
|
34
|
+
"cli/DataMigrator.d.ts": {
|
|
35
|
+
"size": 4570,
|
|
36
|
+
"sha256": "117dd6aa7027816ab95028ff2c398fa3e19b638401d0d33891305a9de4310078"
|
|
37
|
+
},
|
|
38
|
+
"cli/DataMigrator.d.ts.map": {
|
|
39
|
+
"size": 2930,
|
|
40
|
+
"sha256": "4b1a656b6f320f3ee7855358d11e0d51cb89db78d9a258056ca4b4cb2e5b839e"
|
|
41
|
+
},
|
|
42
|
+
"cli/DataMigrator.js": {
|
|
43
|
+
"size": 65681,
|
|
44
|
+
"sha256": "684658568197148c8fa6bbf27f8827e27e2b9b74a8e3b6ecf5a895d9e1a1107e"
|
|
45
|
+
},
|
|
46
|
+
"cli/MigrateToD1Command.d.ts": {
|
|
47
|
+
"size": 1589,
|
|
48
|
+
"sha256": "c1bffb405e91cc1ff61507a51bf80baaeb89ce5a1a98ae591ecaa8e35450a8d2"
|
|
49
|
+
},
|
|
50
|
+
"cli/MigrateToD1Command.d.ts.map": {
|
|
51
|
+
"size": 1235,
|
|
52
|
+
"sha256": "3142b9db2f22130cc97300cb4fac5b4385c2f632c6c35244c96575de2c3d1cab"
|
|
53
|
+
},
|
|
54
|
+
"cli/MigrateToD1Command.js": {
|
|
55
|
+
"size": 36287,
|
|
56
|
+
"sha256": "72e3d20f16fca25ec5f94b96b419953a2cb7f67b56f8726967bda1bc1c808f5b"
|
|
57
|
+
},
|
|
58
|
+
"cli/ProgressTracker.d.ts": {
|
|
59
|
+
"size": 942,
|
|
60
|
+
"sha256": "653253439befb42b452a743588253f6156bb8af1953eadc67f27a4a7ab8cff3d"
|
|
61
|
+
},
|
|
62
|
+
"cli/ProgressTracker.d.ts.map": {
|
|
63
|
+
"size": 467,
|
|
64
|
+
"sha256": "7d38bb24d3744f54059534f4d3c935ac21e1e3ed54e2b4d68af6326e5893a3a0"
|
|
65
|
+
},
|
|
66
|
+
"cli/ProgressTracker.js": {
|
|
67
|
+
"size": 3250,
|
|
68
|
+
"sha256": "1ee31ec2e31446aadf1f0f25e1f55ad63692589d5cbc6c464e571237bb9a3de3"
|
|
69
|
+
},
|
|
70
|
+
"cli/SchemaAnalyzer.d.ts": {
|
|
71
|
+
"size": 4672,
|
|
72
|
+
"sha256": "f06ed184b5ed194d32bcc59988e767ac501d9395d2996aa6a2bd5312a7ebd43d"
|
|
73
|
+
},
|
|
74
|
+
"cli/SchemaAnalyzer.d.ts.map": {
|
|
75
|
+
"size": 2853,
|
|
76
|
+
"sha256": "77971c08d8161dbd52f09bdea1c63aa1e0f8e5598879bdacda1aaa1b05f5057b"
|
|
77
|
+
},
|
|
78
|
+
"cli/SchemaAnalyzer.js": {
|
|
79
|
+
"size": 28767,
|
|
80
|
+
"sha256": "86c9a6d8a17829a7ae45cb99358db343ecd44b7600bab04b8c029cb0e1083948"
|
|
81
|
+
},
|
|
82
|
+
"index.d.ts": {
|
|
83
|
+
"size": 1913,
|
|
84
|
+
"sha256": "1cc13fa32042969f2f426fb3c2971049df657bdbbda8275562ab72395fd2426b"
|
|
85
|
+
},
|
|
86
|
+
"index.d.ts.map": {
|
|
87
|
+
"size": 1564,
|
|
88
|
+
"sha256": "685c4e8b49c8a86f53feacb7c42e3b8268b9402ae3ae7e47cb46e7b78a9bf78b"
|
|
89
|
+
},
|
|
90
|
+
"index.js": {
|
|
91
|
+
"size": 1262,
|
|
92
|
+
"sha256": "add0b0361bb746772b69ef66fbf5544b5da0fff03874448f402e4bf664dedae6"
|
|
93
|
+
},
|
|
94
|
+
"register.d.ts": {
|
|
95
|
+
"size": 159,
|
|
96
|
+
"sha256": "e60a21d17e51095a4a8a9c4a2fae45c902305635998782b8194eb968d8083cb2"
|
|
97
|
+
},
|
|
98
|
+
"register.d.ts.map": {
|
|
99
|
+
"size": 211,
|
|
100
|
+
"sha256": "6743ba50c90440882096f5eaf09a2fff4c901ba238c604d2240ba072b5754632"
|
|
101
|
+
},
|
|
102
|
+
"register.js": {
|
|
103
|
+
"size": 857,
|
|
104
|
+
"sha256": "949f1d93c92a2b9ec1e7d388b4723f8f7012e57eaded61589d205709f0f7a5fe"
|
|
105
|
+
},
|
|
106
|
+
"schema/SchemaBuilder.d.ts": {
|
|
107
|
+
"size": 1523,
|
|
108
|
+
"sha256": "efdf6d8b601de9ccf222be2c67c3730c1c714677f584ba6931225fda7f009f02"
|
|
109
|
+
},
|
|
110
|
+
"schema/SchemaBuilder.d.ts.map": {
|
|
111
|
+
"size": 712,
|
|
112
|
+
"sha256": "3efcc6688f45364eb79d1400493df4ad7251109af87ab2bee9c3ca9e13eb2053"
|
|
113
|
+
},
|
|
114
|
+
"schema/SchemaBuilder.js": {
|
|
115
|
+
"size": 8471,
|
|
116
|
+
"sha256": "ae18b2cdf40970d082e8645ee33446eb29f9634477f4010c881b5842053dd998"
|
|
117
|
+
},
|
|
118
|
+
"schema/TypeConverter.d.ts": {
|
|
119
|
+
"size": 989,
|
|
120
|
+
"sha256": "cca9b405c198b91062c4f44bba7acb0a36dedbca0cb3f523ba576eb5fa375efd"
|
|
121
|
+
},
|
|
122
|
+
"schema/TypeConverter.d.ts.map": {
|
|
123
|
+
"size": 431,
|
|
124
|
+
"sha256": "cef867ebdec5573d7199095d074e55d4222de7516eabaff1913ea377abca6a71"
|
|
125
|
+
},
|
|
126
|
+
"schema/TypeConverter.js": {
|
|
127
|
+
"size": 5851,
|
|
128
|
+
"sha256": "c86af8f57c3e2817d56877c7ad791846de511ac35a753a0e15987515c847de18"
|
|
129
|
+
},
|
|
130
|
+
"schema/Validator.d.ts": {
|
|
131
|
+
"size": 2129,
|
|
132
|
+
"sha256": "3970191b0e8e2d1e19ae60303d2d549cc7a4cc2c7ec24b1b41878adb68354146"
|
|
133
|
+
},
|
|
134
|
+
"schema/Validator.d.ts.map": {
|
|
135
|
+
"size": 1206,
|
|
136
|
+
"sha256": "f6c526a78303423112e7916093bbeed2d7355ab5db5f74ce2c84e56ca9e82837"
|
|
137
|
+
},
|
|
138
|
+
"schema/Validator.js": {
|
|
139
|
+
"size": 8868,
|
|
140
|
+
"sha256": "e8349ddbbf3c969468d741e634a50422847052f21afa38e63493853273945e6e"
|
|
141
|
+
},
|
|
142
|
+
"types.d.ts": {
|
|
143
|
+
"size": 4402,
|
|
144
|
+
"sha256": "a1f52916b14ceed1090ada78df6a23c7353cf9d1a058b5910b9ab6b0d43e9221"
|
|
145
|
+
},
|
|
146
|
+
"types.d.ts.map": {
|
|
147
|
+
"size": 4437,
|
|
148
|
+
"sha256": "63742f1d257f91e6de7ed97a0613addfb1eb67d4794e974c0aa513f38651a2e4"
|
|
149
|
+
},
|
|
150
|
+
"types.js": {
|
|
151
|
+
"size": 85,
|
|
152
|
+
"sha256": "f3a7356e481968927c5e7e784706d8d88a0fa07a742b72216751d6ef0016eb48"
|
|
153
|
+
},
|
|
154
|
+
"utils/CheckpointManager.d.ts": {
|
|
155
|
+
"size": 1408,
|
|
156
|
+
"sha256": "4b0d63804ff4a4013114903f64b65e17f7f184cfc302bfb50d08a41dc68077bb"
|
|
157
|
+
},
|
|
158
|
+
"utils/CheckpointManager.d.ts.map": {
|
|
159
|
+
"size": 736,
|
|
160
|
+
"sha256": "01acea9963980d5a1b27d65531fbf4ab9f1786ad443e9b353faf6ad6adf676d8"
|
|
161
|
+
},
|
|
162
|
+
"utils/CheckpointManager.js": {
|
|
163
|
+
"size": 7381,
|
|
164
|
+
"sha256": "05f143813293dfe3761a682762b13ab650a9a9b9e2d0a208615077d8a2e7d380"
|
|
165
|
+
},
|
|
166
|
+
"utils/DataValidator.d.ts": {
|
|
167
|
+
"size": 1512,
|
|
168
|
+
"sha256": "b7d9127f9d12a7705c454173e6b8c7e6aea66a1143399f7eed9708e37dce2aa3"
|
|
169
|
+
},
|
|
170
|
+
"utils/DataValidator.d.ts.map": {
|
|
171
|
+
"size": 781,
|
|
172
|
+
"sha256": "bfbff6541c2232782cd82baa04492949c4420155b187ceb2dbaa6905dc06118b"
|
|
173
|
+
},
|
|
174
|
+
"utils/DataValidator.js": {
|
|
175
|
+
"size": 5344,
|
|
176
|
+
"sha256": "30f15a50b0f41615ed39d142fd24893a065e052c7644973f7139d1226514eb2e"
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -30,6 +30,7 @@ export interface TargetConnection {
|
|
|
30
30
|
connected: boolean;
|
|
31
31
|
adapter?: DatabaseAdapter;
|
|
32
32
|
remoteBatchTuning?: RemoteBatchTuning;
|
|
33
|
+
targetRowCountsCache?: Record<string, number>;
|
|
33
34
|
}
|
|
34
35
|
export interface TableInfo {
|
|
35
36
|
name: string;
|
|
@@ -83,6 +84,10 @@ export declare const DataMigrator: Readonly<{
|
|
|
83
84
|
getSchemaInfo(_connection: SourceConnection): Promise<{
|
|
84
85
|
tables: TableInfo[];
|
|
85
86
|
}>;
|
|
87
|
+
/**
|
|
88
|
+
* Populate target table row counts cache to optimize performance and prevent countless individual DB count queries
|
|
89
|
+
*/
|
|
90
|
+
populateTargetRowCountsCache(targetConnection: TargetConnection, tables: TableInfo[]): Promise<void>;
|
|
86
91
|
/**
|
|
87
92
|
* Get target table row count for resumability
|
|
88
93
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"DataMigrator.d.ts","sourceRoot":"","sources":["../../src/cli/DataMigrator.ts"],"names":[],"mappings":"AACA;;;GAGG;AAeH,OAAO,KAAK,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC,cAAc,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,CAAC,EAAE,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACnE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;CAC7C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AASD,KAAK,iBAAiB,GAAG;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAQF,KAAK,kBAAkB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAglDF;;;GAGG;AACH,eAAO,MAAM,YAAY;IACvB;;OAEG;wBACuB,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqFtE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAiCzE;;OAEG;4BAC2B,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6CzE;;OAEG;0CAEiB,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC,IAAI,CAAC;IAqFhB;;OAEG;+BAC8B,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC;IAoBpF;;OAEG;mDAEiB,gBAAgB,UAC1B,SAAS,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAgChB;;OAEG;wCACuC,gBAAgB,aAAa,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuB/F;;OAEG;wBAEM,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,UAC1B,eAAe,GACtB,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;8BA0E7C,SAAS,oBACE,gBAAgB,oBAChB,gBAAgB,aACvB,MAAM,aACN,MAAM,eACJ,MAAM,UACX,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,CAAC;IA8BlB;;OAEG;oCAEiB,gBAAgB,aACvB,MAAM,UACT,MAAM,aACH,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA8BrC;;OAEG;yBAEM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,cACpB,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IA0CrC;;OAEG;iCAEiB,gBAAgB,aACvB,MAAM,QACX,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAC9B,OAAO,CAAC,MAAM,CAAC;IAuDlB;;OAEG;gCACyB,eAAe,CAAC,cAAc,CAAC,aAAa,MAAM,GAAG,MAAM;IAavF;;OAEG;wCAEM,MAAM,UACL,MAAM,gBACA,MAAM,gBACN,MAAM,GACnB,0BAA0B;IAS7B;;OAEG;gCACyB,MAAM,GAAG,iBAAiB;IAetD;;OAEG;6BAES,iBAAiB,WAClB,OAAO,CAAC,iBAAiB,CAAC,GAClC,iBAAiB;EAGpB,CAAC"}
|
package/dist/cli/DataMigrator.js
CHANGED
|
@@ -3,8 +3,10 @@
|
|
|
3
3
|
* Data Migrator
|
|
4
4
|
* Handles the actual data migration between databases
|
|
5
5
|
*/
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { LocalD1Resolver, WranglerD1 } from '@zintrust/core/cli';
|
|
7
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
8
|
+
import { Logger } from '@zintrust/core/logger';
|
|
9
|
+
import { appendFileSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
8
10
|
import * as path from 'node:path';
|
|
9
11
|
import { MySQLAdapter } from '@zintrust/db-mysql';
|
|
10
12
|
import { PostgreSQLAdapter } from '@zintrust/db-postgres';
|
|
@@ -519,6 +521,153 @@ const buildTableMigrationLevels = (tables) => {
|
|
|
519
521
|
}
|
|
520
522
|
return levels;
|
|
521
523
|
};
|
|
524
|
+
function groupTablesByLimits(tables) {
|
|
525
|
+
const isGroupingEnabled = (process.env['MIGRATE_TO_D1_GROUP_SMALL_TABLES'] ?? 'true') === 'true';
|
|
526
|
+
if (!isGroupingEnabled) {
|
|
527
|
+
return { largeTables: tables, groups: [] };
|
|
528
|
+
}
|
|
529
|
+
const maxGroupRows = Number.parseInt(process.env['MIGRATE_TO_D1_MAX_GROUP_ROWS'] ?? '50000', 10);
|
|
530
|
+
const maxGroupSizeBytes = Number.parseInt(process.env['MIGRATE_TO_D1_MAX_GROUP_SIZE_MB'] ?? '10', 10) * 1024 * 1024;
|
|
531
|
+
const largeTables = [];
|
|
532
|
+
const groups = [];
|
|
533
|
+
let currentGroup = [];
|
|
534
|
+
let currentGroupRows = 0;
|
|
535
|
+
let currentGroupBytes = 0;
|
|
536
|
+
for (const table of tables) {
|
|
537
|
+
const rowCount = table.rowCount || 0;
|
|
538
|
+
const estimatedBytes = rowCount * 400;
|
|
539
|
+
if (rowCount > maxGroupRows || estimatedBytes > maxGroupSizeBytes) {
|
|
540
|
+
largeTables.push(table);
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
const exceedsRows = currentGroupRows + rowCount > maxGroupRows;
|
|
544
|
+
const exceedsBytes = currentGroupBytes + estimatedBytes > maxGroupSizeBytes;
|
|
545
|
+
if (exceedsRows || exceedsBytes) {
|
|
546
|
+
if (currentGroup.length > 0) {
|
|
547
|
+
groups.push(currentGroup);
|
|
548
|
+
}
|
|
549
|
+
currentGroup = [table];
|
|
550
|
+
currentGroupRows = rowCount;
|
|
551
|
+
currentGroupBytes = estimatedBytes;
|
|
552
|
+
}
|
|
553
|
+
else {
|
|
554
|
+
currentGroup.push(table);
|
|
555
|
+
currentGroupRows += rowCount;
|
|
556
|
+
currentGroupBytes += estimatedBytes;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
if (currentGroup.length > 0) {
|
|
560
|
+
groups.push(currentGroup);
|
|
561
|
+
}
|
|
562
|
+
const finalGroups = groups.filter((group) => {
|
|
563
|
+
if (group.length === 1 && group[0] !== undefined) {
|
|
564
|
+
largeTables.push(group[0]);
|
|
565
|
+
return false;
|
|
566
|
+
}
|
|
567
|
+
return true;
|
|
568
|
+
});
|
|
569
|
+
return { largeTables, groups: finalGroups };
|
|
570
|
+
}
|
|
571
|
+
const processSmallTableRows = async (table, sourceConnection, targetConnection, rowsMigratedMap, allSqlStatements) => {
|
|
572
|
+
const totalRows = table.rowCount || 0;
|
|
573
|
+
const targetRowCount = await DataMigrator.getTargetRowCount(targetConnection, table.name);
|
|
574
|
+
if (targetRowCount >= totalRows) {
|
|
575
|
+
Logger.info(`[DataMigrator] Table ${table.name} already synced: ${targetRowCount}/${totalRows} rows, skipping in group`);
|
|
576
|
+
rowsMigratedMap[table.name] = totalRows;
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
const rows = await DataMigrator.readDataChunk(sourceConnection, table.name, 0, totalRows);
|
|
580
|
+
if (rows.length === 0) {
|
|
581
|
+
rowsMigratedMap[table.name] = 0;
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
const transformedRows = await DataMigrator.transformData(rows, table.name);
|
|
585
|
+
const batchSettings = getInsertBatchSettings(targetConnection);
|
|
586
|
+
const statements = createInsertStatements(targetConnection.type, batchSettings, table.name, transformedRows);
|
|
587
|
+
for (const statement of statements) {
|
|
588
|
+
allSqlStatements.push(bindSqlParameters(statement.sql, statement.parameters));
|
|
589
|
+
}
|
|
590
|
+
rowsMigratedMap[table.name] = rows.length;
|
|
591
|
+
};
|
|
592
|
+
async function migrateSmallTablesGroup(tables, sourceConnection, targetConnection, config) {
|
|
593
|
+
Logger.info(`[DataMigrator] Group migrating ${tables.length} small tables: ${tables.map((t) => t.name).join(', ')}`);
|
|
594
|
+
const errors = [];
|
|
595
|
+
const rowsMigratedMap = {};
|
|
596
|
+
const tempFileDir = path.join('.wrangler', 'tmp');
|
|
597
|
+
const tempFileName = `migration-group-${Date.now()}-${Math.random().toString(36).substring(7)}.sql`;
|
|
598
|
+
const tempFilePath = path.join(tempFileDir, tempFileName);
|
|
599
|
+
try {
|
|
600
|
+
mkdirSync(tempFileDir, { recursive: true });
|
|
601
|
+
const allSqlStatements = [];
|
|
602
|
+
for (const table of tables) {
|
|
603
|
+
await processSmallTableRows(table, sourceConnection, targetConnection, rowsMigratedMap, allSqlStatements);
|
|
604
|
+
}
|
|
605
|
+
if (allSqlStatements.length > 0) {
|
|
606
|
+
const sqlContent = allSqlStatements.join(';\n') + ';';
|
|
607
|
+
writeFileSync(tempFilePath, sqlContent, 'utf8');
|
|
608
|
+
const executionStartTime = Date.now();
|
|
609
|
+
WranglerD1.executeSql({
|
|
610
|
+
dbName: targetConnection.database,
|
|
611
|
+
isLocal: targetConnection.type !== 'd1-remote',
|
|
612
|
+
file: tempFilePath,
|
|
613
|
+
});
|
|
614
|
+
const durationMs = Date.now() - executionStartTime;
|
|
615
|
+
if (targetConnection.targetRowCountsCache) {
|
|
616
|
+
for (const table of tables) {
|
|
617
|
+
targetConnection.targetRowCountsCache[table.name] = table.rowCount || 0;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
Logger.info(`[DataMigrator] Successfully migrated group of ${tables.length} tables in ${formatDuration(durationMs)}`);
|
|
621
|
+
}
|
|
622
|
+
try {
|
|
623
|
+
unlinkSync(tempFilePath);
|
|
624
|
+
}
|
|
625
|
+
catch {
|
|
626
|
+
// Ignore
|
|
627
|
+
}
|
|
628
|
+
return { rowsMigratedMap, errors };
|
|
629
|
+
}
|
|
630
|
+
catch (error) {
|
|
631
|
+
try {
|
|
632
|
+
unlinkSync(tempFilePath);
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
// Ignore
|
|
636
|
+
}
|
|
637
|
+
const errMsg = `Group migration failed: ${getErrorMessage(error)}`;
|
|
638
|
+
Logger.warn(`[DataMigrator] ${errMsg}. Falling back to individual table migration...`);
|
|
639
|
+
for (const table of tables) {
|
|
640
|
+
const result = await DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
|
|
641
|
+
rowsMigratedMap[table.name] = result.rowsMigrated;
|
|
642
|
+
errors.push(...result.errors);
|
|
643
|
+
}
|
|
644
|
+
return { rowsMigratedMap, errors };
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
const migrateTableLevel = async (levelIndex, tables, tableLevels, tableParallelism, sourceConnection, targetConnection, config, progress) => {
|
|
648
|
+
Logger.info(`[DataMigrator] Starting table level ${levelIndex + 1}/${tableLevels.length}: ${tables.map((table) => table.name).join(', ')}`);
|
|
649
|
+
const { largeTables, groups } = groupTablesByLimits(tables);
|
|
650
|
+
const levelResults = await executeWithConcurrency(largeTables, tableParallelism, async (table) => {
|
|
651
|
+
return DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
|
|
652
|
+
});
|
|
653
|
+
for (const [resultIndex, result] of levelResults.entries()) {
|
|
654
|
+
const table = largeTables[resultIndex];
|
|
655
|
+
progress.processedRows += result.rowsMigrated;
|
|
656
|
+
if (result.errors.length > 0 && table !== undefined) {
|
|
657
|
+
progress.errors[table.name] = result.errors.join('; ');
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
for (const group of groups) {
|
|
661
|
+
const groupResult = await migrateSmallTablesGroup(group, sourceConnection, targetConnection, config);
|
|
662
|
+
for (const table of group) {
|
|
663
|
+
const rowsMigrated = groupResult.rowsMigratedMap[table.name] ?? 0;
|
|
664
|
+
progress.processedRows += rowsMigrated;
|
|
665
|
+
if (groupResult.errors.length > 0) {
|
|
666
|
+
progress.errors[table.name] = groupResult.errors.join('; ');
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
};
|
|
522
671
|
const getTableParallelism = (config, targetConnection) => {
|
|
523
672
|
if (targetConnection.type !== 'd1-remote') {
|
|
524
673
|
return 1;
|
|
@@ -611,7 +760,42 @@ const createRemoteInsertExecutionQueue = (targetConnection, tableName, rows) =>
|
|
|
611
760
|
const batchSettings = getInsertBatchSettings(targetConnection);
|
|
612
761
|
return createInsertStatements(targetConnection.type, batchSettings, tableName, rows);
|
|
613
762
|
};
|
|
614
|
-
const
|
|
763
|
+
const executeRemoteFileBatch = async (targetConnection, tableName, statements) => {
|
|
764
|
+
const tempFileDir = path.join('.wrangler', 'tmp');
|
|
765
|
+
const tempFileName = `migration-${tableName}-${Date.now()}-${Math.random().toString(36).substring(7)}.sql`;
|
|
766
|
+
const tempFilePath = path.join(tempFileDir, tempFileName);
|
|
767
|
+
try {
|
|
768
|
+
mkdirSync(tempFileDir, { recursive: true });
|
|
769
|
+
const sqlContent = statements.map((s) => bindSqlParameters(s.sql, s.parameters)).join(';\n') + ';';
|
|
770
|
+
writeFileSync(tempFilePath, sqlContent, 'utf8');
|
|
771
|
+
const executionStartTime = Date.now();
|
|
772
|
+
WranglerD1.executeSql({
|
|
773
|
+
dbName: targetConnection.database,
|
|
774
|
+
isLocal: false,
|
|
775
|
+
file: tempFilePath,
|
|
776
|
+
});
|
|
777
|
+
const durationMs = Date.now() - executionStartTime;
|
|
778
|
+
try {
|
|
779
|
+
unlinkSync(tempFilePath);
|
|
780
|
+
}
|
|
781
|
+
catch {
|
|
782
|
+
// Ignore deletion error
|
|
783
|
+
}
|
|
784
|
+
const affectedRows = statements.reduce((sum, s) => sum + s.rowCount, 0);
|
|
785
|
+
return { affectedRows, durationMs, sqlLength: getSqlByteLength(sqlContent) };
|
|
786
|
+
}
|
|
787
|
+
catch (error) {
|
|
788
|
+
try {
|
|
789
|
+
unlinkSync(tempFilePath);
|
|
790
|
+
}
|
|
791
|
+
catch {
|
|
792
|
+
// Ignore deletion error
|
|
793
|
+
}
|
|
794
|
+
Logger.warn(`[DataMigrator] High-speed SQL file insert failed for ${tableName}, falling back to adaptive statement-by-statement execution. Error: ${getErrorMessage(error)}`);
|
|
795
|
+
return null;
|
|
796
|
+
}
|
|
797
|
+
};
|
|
798
|
+
async function insertRemoteRowsWithRetry(targetConnection, tableName, data) {
|
|
615
799
|
const pendingRows = [data];
|
|
616
800
|
let insertedRows = 0;
|
|
617
801
|
while (pendingRows.length > 0) {
|
|
@@ -621,6 +805,14 @@ const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
|
|
|
621
805
|
}
|
|
622
806
|
const statements = createRemoteInsertExecutionQueue(targetConnection, tableName, currentRows);
|
|
623
807
|
let shouldRetryCurrentRows = false;
|
|
808
|
+
if (targetConnection.type === 'd1-remote' && statements.length > 0) {
|
|
809
|
+
const fileResult = await executeRemoteFileBatch(targetConnection, tableName, statements);
|
|
810
|
+
if (fileResult !== null) {
|
|
811
|
+
insertedRows += fileResult.affectedRows;
|
|
812
|
+
adjustRemoteBatchTuning(targetConnection, fileResult.affectedRows, fileResult.durationMs, fileResult.sqlLength);
|
|
813
|
+
continue;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
624
816
|
for (const [statementIndex, statement] of statements.entries()) {
|
|
625
817
|
try {
|
|
626
818
|
const { affectedRows, executionDurationMs } = await executeInsertStatement(targetConnection, statement);
|
|
@@ -650,7 +842,7 @@ const insertRemoteRowsWithRetry = async (targetConnection, tableName, data) => {
|
|
|
650
842
|
}
|
|
651
843
|
}
|
|
652
844
|
return insertedRows;
|
|
653
|
-
}
|
|
845
|
+
}
|
|
654
846
|
const createRemoteD1Adapter = (database) => {
|
|
655
847
|
return {
|
|
656
848
|
async connect() {
|
|
@@ -688,7 +880,21 @@ const getErrorCause = (error) => {
|
|
|
688
880
|
};
|
|
689
881
|
const getErrorMessage = (error) => {
|
|
690
882
|
if (error instanceof Error) {
|
|
691
|
-
|
|
883
|
+
let msg = error.message;
|
|
884
|
+
const errObj = error;
|
|
885
|
+
if (errObj.stderr && (typeof errObj.stderr === 'string' || Buffer.isBuffer(errObj.stderr))) {
|
|
886
|
+
const stderrStr = errObj.stderr.toString().trim();
|
|
887
|
+
if (stderrStr) {
|
|
888
|
+
msg += `\nStderr: ${stderrStr}`;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
if (errObj.stdout && (typeof errObj.stdout === 'string' || Buffer.isBuffer(errObj.stdout))) {
|
|
892
|
+
const stdoutStr = errObj.stdout.toString().trim();
|
|
893
|
+
if (stdoutStr) {
|
|
894
|
+
msg += `\nStdout: ${stdoutStr}`;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
return msg;
|
|
692
898
|
}
|
|
693
899
|
return String(error);
|
|
694
900
|
};
|
|
@@ -704,6 +910,61 @@ const getErrorChainMessages = (error) => {
|
|
|
704
910
|
}
|
|
705
911
|
return [...new Set(messages)];
|
|
706
912
|
};
|
|
913
|
+
const cacheTargetRowCount = (targetConnection, row) => {
|
|
914
|
+
const name = typeof row['name'] === 'string' ? row['name'] : null;
|
|
915
|
+
const count = Number(row['count']);
|
|
916
|
+
if (name !== null && Number.isFinite(count) && targetConnection.targetRowCountsCache) {
|
|
917
|
+
targetConnection.targetRowCountsCache[name] = count;
|
|
918
|
+
}
|
|
919
|
+
};
|
|
920
|
+
const cacheTargetRowCountPayload = (targetConnection, payload) => {
|
|
921
|
+
for (const statementResult of payload) {
|
|
922
|
+
const row = statementResult.results?.[0];
|
|
923
|
+
if (row !== undefined) {
|
|
924
|
+
cacheTargetRowCount(targetConnection, row);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
};
|
|
928
|
+
const unlinkTempFile = (tempFilePath) => {
|
|
929
|
+
try {
|
|
930
|
+
unlinkSync(tempFilePath);
|
|
931
|
+
}
|
|
932
|
+
catch (error) {
|
|
933
|
+
Logger.debug(`[DataMigrator] Failed to remove temp file ${tempFilePath}: ${getErrorMessage(error)}`);
|
|
934
|
+
}
|
|
935
|
+
};
|
|
936
|
+
const executeRemoteCountPrefetch = (targetConnection, sqlStatements) => {
|
|
937
|
+
const tempFileDir = path.join('.wrangler', 'tmp');
|
|
938
|
+
const tempFileName = `migration-counts-${Date.now()}-${Math.random().toString(36).substring(7)}.sql`;
|
|
939
|
+
const tempFilePath = path.join(tempFileDir, tempFileName);
|
|
940
|
+
try {
|
|
941
|
+
mkdirSync(tempFileDir, { recursive: true });
|
|
942
|
+
writeFileSync(tempFilePath, `${sqlStatements};`, 'utf8');
|
|
943
|
+
const output = WranglerD1.executeSql({
|
|
944
|
+
dbName: targetConnection.database,
|
|
945
|
+
isLocal: false,
|
|
946
|
+
file: tempFilePath,
|
|
947
|
+
});
|
|
948
|
+
const payload = extractWranglerJson(output);
|
|
949
|
+
if (payload !== null) {
|
|
950
|
+
cacheTargetRowCountPayload(targetConnection, payload);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
finally {
|
|
954
|
+
unlinkTempFile(tempFilePath);
|
|
955
|
+
}
|
|
956
|
+
};
|
|
957
|
+
const executeLocalCountPrefetch = async (targetConnection, sqlStatements) => {
|
|
958
|
+
if (!targetConnection.adapter) {
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
961
|
+
for (const sql of sqlStatements) {
|
|
962
|
+
const result = await targetConnection.adapter.query(sql, []);
|
|
963
|
+
for (const row of result.rows) {
|
|
964
|
+
cacheTargetRowCount(targetConnection, row);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
};
|
|
707
968
|
const describeDriverError = (error) => {
|
|
708
969
|
if (error === null || typeof error !== 'object') {
|
|
709
970
|
return undefined;
|
|
@@ -870,22 +1131,13 @@ export const DataMigrator = Object.freeze({
|
|
|
870
1131
|
Logger.info(`Migrating ${progress.totalTables} tables with ${progress.totalRows} total rows`);
|
|
871
1132
|
if (targetConnection.adapter) {
|
|
872
1133
|
await DataMigrator.prepareTargetSchema(sourceConnection, targetConnection, config);
|
|
1134
|
+
await DataMigrator.populateTargetRowCountsCache(targetConnection, schema.tables);
|
|
873
1135
|
}
|
|
874
1136
|
Logger.info('Starting table migration...');
|
|
875
1137
|
const tableLevels = buildTableMigrationLevels(schema.tables);
|
|
876
1138
|
const tableParallelism = getTableParallelism(config, targetConnection);
|
|
877
1139
|
for (const [levelIndex, tables] of tableLevels.entries()) {
|
|
878
|
-
|
|
879
|
-
const levelResults = await executeWithConcurrency(tables, tableParallelism, async (table) => {
|
|
880
|
-
return DataMigrator.migrateTable(table, sourceConnection, targetConnection, config);
|
|
881
|
-
});
|
|
882
|
-
for (const [resultIndex, result] of levelResults.entries()) {
|
|
883
|
-
const table = tables[resultIndex];
|
|
884
|
-
progress.processedRows += result.rowsMigrated;
|
|
885
|
-
if (result.errors.length > 0 && table !== undefined) {
|
|
886
|
-
progress.errors[table.name] = result.errors.join('; ');
|
|
887
|
-
}
|
|
888
|
-
}
|
|
1140
|
+
await migrateTableLevel(levelIndex, tables, tableLevels, tableParallelism, sourceConnection, targetConnection, config, progress);
|
|
889
1141
|
}
|
|
890
1142
|
progress.totalRows = Math.max(progress.totalRows, progress.processedRows);
|
|
891
1143
|
// Update final percentage
|
|
@@ -998,8 +1250,15 @@ export const DataMigrator = Object.freeze({
|
|
|
998
1250
|
if (schemaStatements.length === 0) {
|
|
999
1251
|
return;
|
|
1000
1252
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
1253
|
+
if (targetConnection.type === 'd1-remote') {
|
|
1254
|
+
const batchSql = `${schemaStatements.join(';\n')};`;
|
|
1255
|
+
await adapter.query(batchSql, []);
|
|
1256
|
+
}
|
|
1257
|
+
else {
|
|
1258
|
+
for (const statement of schemaStatements) {
|
|
1259
|
+
await adapter.query(statement, []);
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1003
1262
|
schemaStatements.length = 0;
|
|
1004
1263
|
};
|
|
1005
1264
|
const pushSchemaStatement = async (sql) => {
|
|
@@ -1050,12 +1309,43 @@ export const DataMigrator = Object.freeze({
|
|
|
1050
1309
|
Logger.info(`Found ${tables.length} tables`);
|
|
1051
1310
|
return { tables };
|
|
1052
1311
|
},
|
|
1312
|
+
/**
|
|
1313
|
+
* Populate target table row counts cache to optimize performance and prevent countless individual DB count queries
|
|
1314
|
+
*/
|
|
1315
|
+
async populateTargetRowCountsCache(targetConnection, tables) {
|
|
1316
|
+
if (!targetConnection.adapter)
|
|
1317
|
+
return;
|
|
1318
|
+
targetConnection.targetRowCountsCache = {};
|
|
1319
|
+
if (tables.length === 0)
|
|
1320
|
+
return;
|
|
1321
|
+
Logger.info(`[DataMigrator] Pre-fetching target row counts for ${tables.length} tables to optimize migration speed...`);
|
|
1322
|
+
const chunkSize = 50;
|
|
1323
|
+
for (let i = 0; i < tables.length; i += chunkSize) {
|
|
1324
|
+
const chunk = tables.slice(i, i + chunkSize);
|
|
1325
|
+
const countQueries = chunk.map((t) => `SELECT '${t.name.replaceAll("'", "''")}' as name, COUNT(*) as count FROM \`${t.name}\``);
|
|
1326
|
+
try {
|
|
1327
|
+
if (targetConnection.type === 'd1-remote') {
|
|
1328
|
+
executeRemoteCountPrefetch(targetConnection, countQueries.join(';\n'));
|
|
1329
|
+
}
|
|
1330
|
+
else {
|
|
1331
|
+
await executeLocalCountPrefetch(targetConnection, countQueries);
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
catch (error) {
|
|
1335
|
+
Logger.debug(`[DataMigrator] Chunked pre-fetch failed, falling back to individual table counts on demand. Error: ${getErrorMessage(error)}`);
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
},
|
|
1053
1339
|
/**
|
|
1054
1340
|
* Get target table row count for resumability
|
|
1055
1341
|
*/
|
|
1056
1342
|
async getTargetRowCount(targetConnection, tableName) {
|
|
1057
1343
|
if (!targetConnection.adapter)
|
|
1058
1344
|
return 0;
|
|
1345
|
+
if (targetConnection.targetRowCountsCache &&
|
|
1346
|
+
typeof targetConnection.targetRowCountsCache[tableName] === 'number') {
|
|
1347
|
+
return targetConnection.targetRowCountsCache[tableName];
|
|
1348
|
+
}
|
|
1059
1349
|
try {
|
|
1060
1350
|
const result = await targetConnection.adapter.query(`SELECT COUNT(*) as count FROM \`${tableName}\``, []);
|
|
1061
1351
|
const count = result.rows[0]?.['count'];
|
|
@@ -1206,7 +1496,12 @@ export const DataMigrator = Object.freeze({
|
|
|
1206
1496
|
throw ErrorFactory.createValidationError(`No target adapter configured for ${targetConnection.database}`);
|
|
1207
1497
|
}
|
|
1208
1498
|
if (targetConnection.type === 'd1-remote') {
|
|
1209
|
-
|
|
1499
|
+
const insertedRows = await insertRemoteRowsWithRetry(targetConnection, tableName, data);
|
|
1500
|
+
if (targetConnection.targetRowCountsCache &&
|
|
1501
|
+
typeof targetConnection.targetRowCountsCache[tableName] === 'number') {
|
|
1502
|
+
targetConnection.targetRowCountsCache[tableName] += insertedRows;
|
|
1503
|
+
}
|
|
1504
|
+
return insertedRows;
|
|
1210
1505
|
}
|
|
1211
1506
|
const batchSettings = getInsertBatchSettings(targetConnection);
|
|
1212
1507
|
const statements = createInsertStatements(targetConnection.type, batchSettings, tableName, data);
|
|
@@ -1217,6 +1512,10 @@ export const DataMigrator = Object.freeze({
|
|
|
1217
1512
|
insertedRows += affectedRows;
|
|
1218
1513
|
adjustRemoteBatchTuning(targetConnection, affectedRows, executionDurationMs, getSqlByteLength(statement.sql));
|
|
1219
1514
|
}
|
|
1515
|
+
if (targetConnection.targetRowCountsCache &&
|
|
1516
|
+
typeof targetConnection.targetRowCountsCache[tableName] === 'number') {
|
|
1517
|
+
targetConnection.targetRowCountsCache[tableName] += insertedRows;
|
|
1518
|
+
}
|
|
1220
1519
|
return insertedRows;
|
|
1221
1520
|
},
|
|
1222
1521
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"MigrateToD1Command.d.ts","sourceRoot":"","sources":["../../src/cli/MigrateToD1Command.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAA+B,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EAAE,eAAe,EAA0B,MAAM,UAAU,CAAC;AAOxE,KAAK,iBAAiB,GAAG;IACvB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,IAAI,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CAC/B,CAAC;AAouBF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,iBA6H/B,CAAC;AAEH;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAetE;AAED;;GAEG;AACH,iBAAe,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAmGxE;AAED;;GAEG;AACH,iBAAe,gBAAgB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGtE;AAED;;GAEG;AACH,iBAAS,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CA2BrF;AAGD,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC"}
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* Migrate to D1 Command
|
|
3
3
|
* CLI command for migrating databases to Cloudflare D1
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { BaseCommand, WranglerConfig } from '@zintrust/core/cli';
|
|
6
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
7
|
+
import { Logger } from '@zintrust/core/logger';
|
|
7
8
|
import { SchemaBuilder } from '../schema/SchemaBuilder.js';
|
|
8
9
|
import { SchemaValidator } from '../schema/Validator.js';
|
|
9
10
|
import { DataMigrator } from './DataMigrator.js';
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Progress Tracker
|
|
3
3
|
* Tracks migration progress and provides status updates
|
|
4
4
|
*/
|
|
5
|
-
import { Logger } from '@zintrust/core';
|
|
5
|
+
import { Logger } from '@zintrust/core/logger';
|
|
6
6
|
/**
|
|
7
7
|
* ProgressTracker - Sealed namespace for progress tracking
|
|
8
8
|
* Provides migration progress monitoring and reporting
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SchemaAnalyzer.d.ts","sourceRoot":"","sources":["../../src/cli/SchemaAnalyzer.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"SchemaAnalyzer.d.ts","sourceRoot":"","sources":["../../src/cli/SchemaAnalyzer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,KAAK,EACV,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,eAAe,EACf,iBAAiB,EACjB,WAAW,EACZ,MAAM,UAAU,CAAC;AAoHlB,MAAM,WAAW,gBAAgB;IAC/B,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,KAAK,CACH,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,OAAO,EAAE,GACpB,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IACtF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,WAAW,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAChF,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACzE,qBAAqB,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,OAAO,IAAI,MAAM,CAAC;IAClB,WAAW,IAAI,OAAO,CAAC;IACvB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;CACvC;AAED;;;GAGG;AACH,eAAO,MAAM,cAAc;IACzB;;OAEG;8BAC6B;QAC9B,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;QAChD,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,CAAC;IA4B3B;;OAEG;8BAC6B;QAC9B,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,MAAM,CAAC;QACzB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;QAChD,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA8E1B;;OAEG;sCAEY;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,WAChD,WAAW,EAAE,GACrB,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAoB/B;;OAEG;oCAEY;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,WAChD,WAAW,EAAE,GACrB,OAAO,CAAC,eAAe,EAAE,CAAC;IAwC7B;;OAEG;iCAC0B,cAAc,GAAG;QAC5C,UAAU,EAAE,OAAO,CAAC;QACpB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;KACpB;IA0BD;;OAEG;0BACmB,MAAM,GAAG,OAAO;IAiBtC;;OAEG;2BACoB,MAAM,GAAG,OAAO;IA2BvC;;OAEG;2BACoB,cAAc,GAAG,MAAM;IA0B9C;;OAEG;0BACyB,gBAAgB,UAAU,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAyChF;;OAEG;4BAEQ,gBAAgB,aACd,MAAM,UACT,MAAM,GACb,OAAO,CAAC,WAAW,CAAC;IAoCvB;;OAEG;6BAEQ,gBAAgB,aACd,MAAM,UACT,MAAM,GACb,OAAO,CAAC,YAAY,EAAE,CAAC;IAwE1B;;OAEG;2BAEQ,gBAAgB,aACd,MAAM,UACT,MAAM,GACb,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAkDzB;;OAEG;gCACyB,MAAM,WAAW,MAAM,GAAG,MAAM;IAgC5D;;OAEG;6BAEQ,gBAAgB,aACd,MAAM,UACT,MAAM,GACb,OAAO,CAAC,WAAW,EAAE,CAAC;IAgBzB;;OAEG;+BACwB,MAAM,UAAU,MAAM,GAAG,MAAM,GAAG,IAAI;IA6CjE;;OAEG;gCACyB;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;KAAE,UAAU,MAAM,GAAG,WAAW,EAAE;IAiC/F;;OAEG;uBACgB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,MAAM,GAAG,OAAO;IAapE;;OAEG;4BAEQ,gBAAgB,aACd,MAAM,UACT,MAAM,GACb,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAgB9B;;OAEG;oCAC6B,MAAM,UAAU,MAAM,GAAG,MAAM,GAAG,IAAI;IAiEtE;;OAEG;8BACuB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,MAAM,GAAG,gBAAgB;IA2BvF;;OAEG;iCAC0B,MAAM,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU;EASzE,CAAC"}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Schema Analyzer
|
|
3
3
|
* Analyzes database schemas for migration compatibility
|
|
4
4
|
*/
|
|
5
|
-
import { ErrorFactory
|
|
5
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
6
|
+
import { Logger } from '@zintrust/core/logger';
|
|
6
7
|
import { MySQLAdapter } from '@zintrust/db-mysql';
|
|
7
8
|
import { PostgreSQLAdapter } from '@zintrust/db-postgres';
|
|
8
9
|
import { SQLiteAdapter } from '@zintrust/db-sqlite';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SchemaBuilder.d.ts","sourceRoot":"","sources":["../../src/schema/SchemaBuilder.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"SchemaBuilder.d.ts","sourceRoot":"","sources":["../../src/schema/SchemaBuilder.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AA2B1D;;;GAGG;AACH,eAAO,MAAM,aAAa;IACxB;;OAEG;gCACyB,WAAW,EAAE,gBAAgB,MAAM,GAAG,WAAW,EAAE;IAM/E;;OAEG;8BACuB,WAAW,gBAAgB,MAAM,GAAG,WAAW;IA+BzE;;OAEG;gCACyB,YAAY,gBAAgB,MAAM,GAAG,YAAY;IAoB7E;;OAEG;kCAC2B,WAAW,GAAG,MAAM;IAwBlD;;OAEG;qCAC8B,YAAY,qBAAoB,OAAO,GAAW,MAAM;IAkBzF;;OAEG;8BACuB,OAAO,GAAG,MAAM;IAkB1C;;OAEG;4BACqB,WAAW,GAAG,MAAM,EAAE;IAqB9C;;OAEG;8BACuB,WAAW,EAAE,GAAG,MAAM;IAqBhD;;OAEG;2BACoB,WAAW,EAAE,GAAG;QACrC,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,EAAE,MAAM,EAAE,CAAC;KAClB;8BA+CyB,WAAW,EAAE,GAAG,IAAI;EAU9C,CAAC"}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Schema Builder
|
|
3
3
|
* Builds D1/SQLite compatible schemas from source schemas
|
|
4
4
|
*/
|
|
5
|
-
import { ErrorFactory
|
|
5
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
6
|
+
import { Logger } from '@zintrust/core/logger';
|
|
6
7
|
import { DataValidator } from '../utils/DataValidator.js';
|
|
7
8
|
import { TypeConverter } from './TypeConverter.js';
|
|
8
9
|
const normalizeNullLikeDefaultValue = (value) => {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Type Converter
|
|
3
3
|
* Converts database types between different database systems
|
|
4
4
|
*/
|
|
5
|
-
import { Logger } from '@zintrust/core';
|
|
5
|
+
import { Logger } from '@zintrust/core/logger';
|
|
6
6
|
/**
|
|
7
7
|
* TypeConverter - Sealed namespace for type conversion
|
|
8
8
|
* Provides database type conversion utilities
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CheckpointManager.d.ts","sourceRoot":"","sources":["../../src/utils/CheckpointManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"CheckpointManager.d.ts","sourceRoot":"","sources":["../../src/utils/CheckpointManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAM/E;;;GAGG;AACH,eAAO,MAAM,iBAAiB;IAC5B;;OAEG;qBACoB,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;qBACoB,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAerD;;OAEG;2BAC0B,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAoBpE;;OAEG;+BAC8B,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB/D;;OAEG;gCAC+B,MAAM,SAAS,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAcxF;;OAEG;mCACkC,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuCvE;;OAEG;yBACwB,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBjD;;OAEG;sBACqB,OAAO,CAAC,MAAM,EAAE,CAAC;IAgBzC;;OAEG;0BACyB,MAAM,aAAa,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;EAuB7E,CAAC"}
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
* Checkpoint Manager
|
|
3
3
|
* File-based persistent state tracking for resumable operations
|
|
4
4
|
*/
|
|
5
|
-
import { ErrorFactory
|
|
5
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
6
|
+
import { Logger } from '@zintrust/core/logger';
|
|
7
|
+
import { NodeSingletons } from '@zintrust/core/workers';
|
|
6
8
|
const MIGRATION_DIR = '.zintrust/migration';
|
|
7
9
|
const path = NodeSingletons.path;
|
|
8
10
|
const { fsPromises, mkdir, readFile, rm, writeFile } = NodeSingletons.fs;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Data Validator
|
|
3
3
|
* Integrity checks and validation for migrated data
|
|
4
4
|
*/
|
|
5
|
-
import { Logger } from '@zintrust/core';
|
|
5
|
+
import { Logger } from '@zintrust/core/logger';
|
|
6
6
|
/**
|
|
7
7
|
* DataValidator - Sealed namespace for data integrity validation
|
|
8
8
|
* Provides comprehensive validation of migrated data
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/d1-migrator",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.3",
|
|
4
4
|
"description": "Resumable database migration toolkit for moving data to Cloudflare D1 with ZinTrust.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"type": "module",
|
|
@@ -40,10 +40,10 @@
|
|
|
40
40
|
"prepublishOnly": "npm run build"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@zintrust/db-d1": "
|
|
44
|
-
"@zintrust/db-mysql": "
|
|
45
|
-
"@zintrust/db-postgres": "
|
|
46
|
-
"@zintrust/db-sqlite": "
|
|
47
|
-
"@zintrust/db-sqlserver": "
|
|
43
|
+
"@zintrust/db-d1": "2.0.0",
|
|
44
|
+
"@zintrust/db-mysql": "2.0.3",
|
|
45
|
+
"@zintrust/db-postgres": "2.0.0",
|
|
46
|
+
"@zintrust/db-sqlite": "2.0.0",
|
|
47
|
+
"@zintrust/db-sqlserver": "2.0.0"
|
|
48
48
|
}
|
|
49
|
-
}
|
|
49
|
+
}
|