@fazetitans/fscopy 1.3.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -1
- package/package.json +19 -15
- package/src/cli.ts +14 -9
- package/src/config/defaults.ts +1 -0
- package/src/config/parser.ts +120 -49
- package/src/config/validator.ts +51 -0
- package/src/constants.ts +39 -0
- package/src/firebase/index.ts +29 -18
- package/src/interactive.ts +526 -36
- package/src/orchestrator.ts +100 -34
- package/src/state/index.ts +42 -26
- package/src/transfer/clear.ts +104 -68
- package/src/transfer/count.ts +54 -36
- package/src/transfer/helpers.ts +36 -1
- package/src/transfer/index.ts +7 -1
- package/src/transfer/transfer.ts +181 -167
- package/src/transform/loader.ts +13 -1
- package/src/types.ts +4 -2
- package/src/utils/credentials.ts +7 -4
- package/src/utils/errors.ts +7 -2
- package/src/utils/index.ts +6 -1
- package/src/utils/output.ts +6 -0
- package/src/utils/progress.ts +10 -0
- package/src/webhook/index.ts +127 -44
package/README.md
CHANGED
|
@@ -389,6 +389,7 @@ fscopy --init config.json
|
|
|
389
389
|
| `--max-depth` | | number | `0` | Max subcollection depth (0 = unlimited) |
|
|
390
390
|
| `--detect-conflicts` | | boolean | `false` | Detect concurrent modifications |
|
|
391
391
|
| `--verify-integrity` | | boolean | `false` | Verify document integrity with hash |
|
|
392
|
+
| `--allow-http-webhook` | | boolean | `false` | Allow HTTP (non-HTTPS) webhook URLs |
|
|
392
393
|
|
|
393
394
|
## How It Works
|
|
394
395
|
|
|
@@ -402,7 +403,9 @@ fscopy --init config.json
|
|
|
402
403
|
|
|
403
404
|
- **Transform files execute arbitrary code** - The `--transform` option uses dynamic imports to load and execute JavaScript/TypeScript files. Only use transform files you have written or thoroughly reviewed. Malicious transform files could access your filesystem, network, or credentials.
|
|
404
405
|
|
|
405
|
-
- **Webhook URLs
|
|
406
|
+
- **Webhook URLs must use HTTPS** - fscopy blocks HTTP webhooks by default (except localhost). Use `--allow-http-webhook` to explicitly allow unencrypted webhooks. Webhook payloads contain project names and transfer statistics that could be sensitive.
|
|
407
|
+
|
|
408
|
+
- **Transform file extensions are validated** - Only `.ts`, `.js`, `.mjs`, and `.mts` files are accepted as transform files.
|
|
406
409
|
|
|
407
410
|
- **Credentials via ADC** - fscopy uses Google Application Default Credentials. Ensure you're authenticated with the correct account before running transfers.
|
|
408
411
|
|
|
@@ -416,8 +419,16 @@ fscopy --init config.json
|
|
|
416
419
|
- **Clear is destructive** - `--clear` deletes all destination docs before transfer
|
|
417
420
|
- **Delete-missing syncs** - `--delete-missing` removes orphan docs after transfer
|
|
418
421
|
- **Transform applies to all** - Transform function is applied to both root and subcollection docs
|
|
422
|
+
- **Limit applies to root only** - `--limit` restricts the number of documents at the root collection level; subcollections are always copied in full
|
|
419
423
|
- **Same project allowed** - Source and destination can be the same project when using `--rename-collection` or `--id-prefix`/`--id-suffix`
|
|
420
424
|
|
|
425
|
+
## Environment Variables
|
|
426
|
+
|
|
427
|
+
| Variable | Description |
|
|
428
|
+
| -------- | ----------- |
|
|
429
|
+
| `GOOGLE_APPLICATION_CREDENTIALS` | Path to Google Cloud credentials JSON file |
|
|
430
|
+
| `FSCOPY_SKIP_CREDENTIALS_CHECK` | Set to `1` to skip the credentials check at startup (useful in CI/CD with alternative auth) |
|
|
431
|
+
|
|
421
432
|
## Limitations
|
|
422
433
|
|
|
423
434
|
### Firestore Special Types
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fazetitans/fscopy",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Fast CLI tool to copy Firestore collections between Firebase projects with filtering, parallel transfers, and subcollection support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -9,15 +9,18 @@
|
|
|
9
9
|
"scripts": {
|
|
10
10
|
"start": "bun src/cli.ts",
|
|
11
11
|
"dev": "bun --watch src/cli.ts",
|
|
12
|
-
"test": "bun test",
|
|
12
|
+
"test": "bun test src/__tests__/*.test.ts",
|
|
13
|
+
"test:integration": "firebase emulators:exec --only firestore --project fscopy-test 'FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 FSCOPY_SKIP_CREDENTIALS_CHECK=1 bun test src/__tests__/integration/'",
|
|
14
|
+
"test:all": "bun run test && bun run test:integration",
|
|
13
15
|
"test:watch": "bun test --watch",
|
|
14
|
-
"test:coverage": "bun test --coverage",
|
|
16
|
+
"test:coverage": "bun test --coverage src/__tests__/*.test.ts",
|
|
17
|
+
"emulator": "firebase emulators:start --only firestore --project fscopy-test",
|
|
15
18
|
"type-check": "tsc --noEmit",
|
|
16
19
|
"lint": "eslint src/**/*.ts --no-warn-ignored",
|
|
17
20
|
"lint:fix": "eslint src/**/*.ts --no-warn-ignored --fix",
|
|
18
21
|
"format": "prettier --write src/**/*.ts",
|
|
19
22
|
"format:check": "prettier --check src/**/*.ts",
|
|
20
|
-
"prepublishOnly": "bun run type-check && bun run lint && bun test"
|
|
23
|
+
"prepublishOnly": "bun run type-check && bun run lint && bun run test"
|
|
21
24
|
},
|
|
22
25
|
"keywords": [
|
|
23
26
|
"firestore",
|
|
@@ -54,24 +57,25 @@
|
|
|
54
57
|
"LICENSE"
|
|
55
58
|
],
|
|
56
59
|
"dependencies": {
|
|
57
|
-
"@inquirer/prompts": "^8.
|
|
60
|
+
"@inquirer/prompts": "^8.3.0",
|
|
58
61
|
"cli-progress": "^3.12.0",
|
|
59
|
-
"firebase-admin": "^13.
|
|
62
|
+
"firebase-admin": "^13.7.0",
|
|
60
63
|
"ini": "^6.0.0",
|
|
61
|
-
"yargs": "^
|
|
64
|
+
"yargs": "^18.0.0"
|
|
62
65
|
},
|
|
63
66
|
"devDependencies": {
|
|
64
|
-
"@eslint/js": "^
|
|
67
|
+
"@eslint/js": "^10.0.1",
|
|
65
68
|
"@types/cli-progress": "^3.11.6",
|
|
66
69
|
"@types/ini": "^4.1.1",
|
|
67
|
-
"@types/node": "^25.
|
|
70
|
+
"@types/node": "^25.3.5",
|
|
68
71
|
"@types/yargs": "^17.0.35",
|
|
69
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
70
|
-
"@typescript-eslint/parser": "^8.
|
|
71
|
-
"bun-types": "^1.3.
|
|
72
|
-
"eslint": "^
|
|
73
|
-
"prettier": "^3.
|
|
72
|
+
"@typescript-eslint/eslint-plugin": "^8.56.1",
|
|
73
|
+
"@typescript-eslint/parser": "^8.56.1",
|
|
74
|
+
"bun-types": "^1.3.10",
|
|
75
|
+
"eslint": "^10.0.2",
|
|
76
|
+
"prettier": "^3.8.1",
|
|
74
77
|
"typescript": "^5.9.3",
|
|
75
|
-
"
|
|
78
|
+
"firebase-tools": "^15.9.0",
|
|
79
|
+
"typescript-eslint": "^8.56.1"
|
|
76
80
|
}
|
|
77
81
|
}
|
package/src/cli.ts
CHANGED
|
@@ -61,7 +61,7 @@ const argv = yargs(hideBin(process.argv))
|
|
|
61
61
|
.option('limit', {
|
|
62
62
|
alias: 'l',
|
|
63
63
|
type: 'number',
|
|
64
|
-
description: 'Limit number of documents per collection (0 = no limit)',
|
|
64
|
+
description: 'Limit number of documents per root collection (0 = no limit, does not apply to subcollections)',
|
|
65
65
|
})
|
|
66
66
|
.option('source-project', {
|
|
67
67
|
type: 'string',
|
|
@@ -89,7 +89,6 @@ const argv = yargs(hideBin(process.argv))
|
|
|
89
89
|
.option('retries', {
|
|
90
90
|
type: 'number',
|
|
91
91
|
description: 'Number of retries on error (default: 3)',
|
|
92
|
-
default: 3,
|
|
93
92
|
})
|
|
94
93
|
.option('quiet', {
|
|
95
94
|
alias: 'q',
|
|
@@ -166,17 +165,14 @@ const argv = yargs(hideBin(process.argv))
|
|
|
166
165
|
.option('verify', {
|
|
167
166
|
type: 'boolean',
|
|
168
167
|
description: 'Verify document counts after transfer',
|
|
169
|
-
default: false,
|
|
170
168
|
})
|
|
171
169
|
.option('rate-limit', {
|
|
172
170
|
type: 'number',
|
|
173
171
|
description: 'Limit transfer rate (documents per second, 0 = unlimited)',
|
|
174
|
-
default: 0,
|
|
175
172
|
})
|
|
176
173
|
.option('skip-oversized', {
|
|
177
174
|
type: 'boolean',
|
|
178
175
|
description: 'Skip documents exceeding 1MB instead of failing',
|
|
179
|
-
default: false,
|
|
180
176
|
})
|
|
181
177
|
.option('json', {
|
|
182
178
|
type: 'boolean',
|
|
@@ -191,16 +187,18 @@ const argv = yargs(hideBin(process.argv))
|
|
|
191
187
|
.option('detect-conflicts', {
|
|
192
188
|
type: 'boolean',
|
|
193
189
|
description: 'Detect if destination docs were modified during transfer',
|
|
194
|
-
default: false,
|
|
195
190
|
})
|
|
196
191
|
.option('max-depth', {
|
|
197
192
|
type: 'number',
|
|
198
193
|
description: 'Max subcollection depth (0 = unlimited)',
|
|
199
|
-
default: 0,
|
|
200
194
|
})
|
|
201
195
|
.option('verify-integrity', {
|
|
202
196
|
type: 'boolean',
|
|
203
197
|
description: 'Verify document integrity with hash after transfer',
|
|
198
|
+
})
|
|
199
|
+
.option('allow-http-webhook', {
|
|
200
|
+
type: 'boolean',
|
|
201
|
+
description: 'Allow webhook URLs using HTTP instead of HTTPS',
|
|
204
202
|
default: false,
|
|
205
203
|
})
|
|
206
204
|
.option('validate-only', {
|
|
@@ -250,7 +248,11 @@ async function main(): Promise<void> {
|
|
|
250
248
|
|
|
251
249
|
// Run interactive mode if enabled
|
|
252
250
|
if (argv.interactive) {
|
|
253
|
-
|
|
251
|
+
const result = await runInteractiveMode(config);
|
|
252
|
+
config = result.config;
|
|
253
|
+
if (result.action === 'save') {
|
|
254
|
+
process.exit(0);
|
|
255
|
+
}
|
|
254
256
|
}
|
|
255
257
|
|
|
256
258
|
displayConfig(config);
|
|
@@ -272,7 +274,10 @@ async function main(): Promise<void> {
|
|
|
272
274
|
|
|
273
275
|
// Validate webhook URL if configured
|
|
274
276
|
if (validConfig.webhook) {
|
|
275
|
-
const webhookValidation = validateWebhookUrl(
|
|
277
|
+
const webhookValidation = validateWebhookUrl(
|
|
278
|
+
validConfig.webhook,
|
|
279
|
+
validConfig.allowHttpWebhook
|
|
280
|
+
);
|
|
276
281
|
if (!webhookValidation.valid) {
|
|
277
282
|
console.log(`\n❌ ${webhookValidation.warning}`);
|
|
278
283
|
process.exit(1);
|
package/src/config/defaults.ts
CHANGED
package/src/config/parser.ts
CHANGED
|
@@ -3,9 +3,18 @@ import path from 'node:path';
|
|
|
3
3
|
import ini from 'ini';
|
|
4
4
|
import type { Config, WhereFilter, CliArgs } from '../types.js';
|
|
5
5
|
|
|
6
|
+
const SUPPORTED_EXTENSIONS = new Set(['.ini', '.json', '.cfg', '.conf']);
|
|
7
|
+
|
|
8
|
+
function stderrWarn(message: string): void {
|
|
9
|
+
process.stderr.write(`${message}\n`);
|
|
10
|
+
}
|
|
11
|
+
|
|
6
12
|
export function getFileFormat(filePath: string): 'json' | 'ini' {
|
|
7
13
|
const ext = path.extname(filePath).toLowerCase();
|
|
8
14
|
if (ext === '.json') return 'json';
|
|
15
|
+
if (ext && !SUPPORTED_EXTENSIONS.has(ext)) {
|
|
16
|
+
stderrWarn(`⚠️ Unrecognized config file extension "${ext}", parsing as INI format`);
|
|
17
|
+
}
|
|
9
18
|
return 'ini';
|
|
10
19
|
}
|
|
11
20
|
|
|
@@ -22,7 +31,7 @@ export function parseWhereFilter(filterStr: string): WhereFilter | null {
|
|
|
22
31
|
const match = new RegExp(operatorRegex).exec(filterStr);
|
|
23
32
|
|
|
24
33
|
if (!match) {
|
|
25
|
-
|
|
34
|
+
stderrWarn(`⚠️ Invalid where filter: "${filterStr}" (missing operator)`);
|
|
26
35
|
return null;
|
|
27
36
|
}
|
|
28
37
|
|
|
@@ -30,20 +39,20 @@ export function parseWhereFilter(filterStr: string): WhereFilter | null {
|
|
|
30
39
|
const [fieldPart, valuePart] = filterStr.split(operatorRegex).filter((_, i) => i !== 1);
|
|
31
40
|
|
|
32
41
|
if (!fieldPart || !valuePart) {
|
|
33
|
-
|
|
42
|
+
stderrWarn(`⚠️ Invalid where filter: "${filterStr}" (missing field or value)`);
|
|
34
43
|
return null;
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
const field = fieldPart.trim();
|
|
38
47
|
const rawValue = valuePart.trim();
|
|
39
48
|
|
|
40
|
-
let value: string | number | boolean;
|
|
49
|
+
let value: string | number | boolean | null;
|
|
41
50
|
if (rawValue === 'true') {
|
|
42
51
|
value = true;
|
|
43
52
|
} else if (rawValue === 'false') {
|
|
44
53
|
value = false;
|
|
45
54
|
} else if (rawValue === 'null') {
|
|
46
|
-
value = null
|
|
55
|
+
value = null;
|
|
47
56
|
} else if (!Number.isNaN(Number(rawValue)) && rawValue !== '') {
|
|
48
57
|
value = Number(rawValue);
|
|
49
58
|
} else {
|
|
@@ -66,6 +75,12 @@ export function parseStringList(value: string | undefined): string[] {
|
|
|
66
75
|
.filter((s) => s.length > 0);
|
|
67
76
|
}
|
|
68
77
|
|
|
78
|
+
export function parseIntOption(value: string | undefined): number | undefined {
|
|
79
|
+
if (value === undefined || value === '') return undefined;
|
|
80
|
+
const parsed = Number.parseInt(value, 10);
|
|
81
|
+
return Number.isNaN(parsed) ? undefined : parsed;
|
|
82
|
+
}
|
|
83
|
+
|
|
69
84
|
export function parseRenameMapping(
|
|
70
85
|
mappings: string[] | string | undefined
|
|
71
86
|
): Record<string, string> {
|
|
@@ -78,13 +93,13 @@ export function parseRenameMapping(
|
|
|
78
93
|
const mapping = String(item).trim();
|
|
79
94
|
const colonIndex = mapping.indexOf(':');
|
|
80
95
|
if (colonIndex === -1) {
|
|
81
|
-
|
|
96
|
+
stderrWarn(`⚠️ Invalid rename mapping: "${mapping}" (missing ':')`);
|
|
82
97
|
continue;
|
|
83
98
|
}
|
|
84
99
|
const source = mapping.slice(0, colonIndex).trim();
|
|
85
100
|
const dest = mapping.slice(colonIndex + 1).trim();
|
|
86
101
|
if (!source || !dest) {
|
|
87
|
-
|
|
102
|
+
stderrWarn(`⚠️ Invalid rename mapping: "${mapping}" (empty source or dest)`);
|
|
88
103
|
continue;
|
|
89
104
|
}
|
|
90
105
|
result[source] = dest;
|
|
@@ -115,6 +130,13 @@ export function parseIniConfig(content: string): Partial<Config> {
|
|
|
115
130
|
idPrefix?: string;
|
|
116
131
|
idSuffix?: string;
|
|
117
132
|
webhook?: string;
|
|
133
|
+
retries?: string;
|
|
134
|
+
rateLimit?: string;
|
|
135
|
+
skipOversized?: string | boolean;
|
|
136
|
+
detectConflicts?: string | boolean;
|
|
137
|
+
maxDepth?: string;
|
|
138
|
+
verify?: string | boolean;
|
|
139
|
+
verifyIntegrity?: string | boolean;
|
|
118
140
|
};
|
|
119
141
|
};
|
|
120
142
|
|
|
@@ -149,6 +171,23 @@ export function parseIniConfig(content: string): Partial<Config> {
|
|
|
149
171
|
idPrefix: parsed.options?.idPrefix ?? null,
|
|
150
172
|
idSuffix: parsed.options?.idSuffix ?? null,
|
|
151
173
|
webhook: parsed.options?.webhook ?? null,
|
|
174
|
+
retries: parseIntOption(parsed.options?.retries),
|
|
175
|
+
rateLimit: parseIntOption(parsed.options?.rateLimit),
|
|
176
|
+
skipOversized:
|
|
177
|
+
parsed.options?.skipOversized !== undefined
|
|
178
|
+
? parseBoolean(parsed.options.skipOversized)
|
|
179
|
+
: undefined,
|
|
180
|
+
detectConflicts:
|
|
181
|
+
parsed.options?.detectConflicts !== undefined
|
|
182
|
+
? parseBoolean(parsed.options.detectConflicts)
|
|
183
|
+
: undefined,
|
|
184
|
+
maxDepth: parseIntOption(parsed.options?.maxDepth),
|
|
185
|
+
verify:
|
|
186
|
+
parsed.options?.verify !== undefined ? parseBoolean(parsed.options.verify) : undefined,
|
|
187
|
+
verifyIntegrity:
|
|
188
|
+
parsed.options?.verifyIntegrity !== undefined
|
|
189
|
+
? parseBoolean(parsed.options.verifyIntegrity)
|
|
190
|
+
: undefined,
|
|
152
191
|
};
|
|
153
192
|
}
|
|
154
193
|
|
|
@@ -172,6 +211,13 @@ export function parseJsonConfig(content: string): Partial<Config> {
|
|
|
172
211
|
idPrefix?: string;
|
|
173
212
|
idSuffix?: string;
|
|
174
213
|
webhook?: string;
|
|
214
|
+
retries?: number;
|
|
215
|
+
rateLimit?: number;
|
|
216
|
+
skipOversized?: boolean;
|
|
217
|
+
detectConflicts?: boolean;
|
|
218
|
+
maxDepth?: number;
|
|
219
|
+
verify?: boolean;
|
|
220
|
+
verifyIntegrity?: boolean;
|
|
175
221
|
};
|
|
176
222
|
|
|
177
223
|
return {
|
|
@@ -193,6 +239,13 @@ export function parseJsonConfig(content: string): Partial<Config> {
|
|
|
193
239
|
idPrefix: config.idPrefix ?? null,
|
|
194
240
|
idSuffix: config.idSuffix ?? null,
|
|
195
241
|
webhook: config.webhook ?? null,
|
|
242
|
+
retries: config.retries,
|
|
243
|
+
rateLimit: config.rateLimit,
|
|
244
|
+
skipOversized: config.skipOversized,
|
|
245
|
+
detectConflicts: config.detectConflicts,
|
|
246
|
+
maxDepth: config.maxDepth,
|
|
247
|
+
verify: config.verify,
|
|
248
|
+
verifyIntegrity: config.verifyIntegrity,
|
|
196
249
|
};
|
|
197
250
|
}
|
|
198
251
|
|
|
@@ -207,11 +260,23 @@ export function loadConfigFile(configPath?: string): Partial<Config> {
|
|
|
207
260
|
const content = fs.readFileSync(absolutePath, 'utf-8');
|
|
208
261
|
const format = getFileFormat(absolutePath);
|
|
209
262
|
|
|
210
|
-
|
|
263
|
+
process.stderr.write(`📄 Loaded config from: ${absolutePath} (${format.toUpperCase()})\n`);
|
|
211
264
|
|
|
212
265
|
return format === 'json' ? parseJsonConfig(content) : parseIniConfig(content);
|
|
213
266
|
}
|
|
214
267
|
|
|
268
|
+
/**
|
|
269
|
+
* Resolve a config value with priority: CLI > file > default.
|
|
270
|
+
*/
|
|
271
|
+
function resolve<K extends keyof Config>(
|
|
272
|
+
key: K,
|
|
273
|
+
cliArgs: Partial<Record<K, Config[K]>>,
|
|
274
|
+
fileConfig: Partial<Config>,
|
|
275
|
+
defaultConfig: Config
|
|
276
|
+
): Config[K] {
|
|
277
|
+
return (cliArgs[key] ?? fileConfig[key] ?? defaultConfig[key]) as Config[K];
|
|
278
|
+
}
|
|
279
|
+
|
|
215
280
|
export function mergeConfig(
|
|
216
281
|
defaultConfig: Config,
|
|
217
282
|
fileConfig: Partial<Config>,
|
|
@@ -220,46 +285,52 @@ export function mergeConfig(
|
|
|
220
285
|
const cliWhereFilters = parseWhereFilters(cliArgs.where);
|
|
221
286
|
const cliRenameCollection = parseRenameMapping(cliArgs.renameCollection);
|
|
222
287
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
288
|
+
// Keys that follow standard CLI > file > default priority
|
|
289
|
+
const standardKeys = [
|
|
290
|
+
'collections',
|
|
291
|
+
'includeSubcollections',
|
|
292
|
+
'dryRun',
|
|
293
|
+
'batchSize',
|
|
294
|
+
'limit',
|
|
295
|
+
'sourceProject',
|
|
296
|
+
'destProject',
|
|
297
|
+
'exclude',
|
|
298
|
+
'merge',
|
|
299
|
+
'parallel',
|
|
300
|
+
'clear',
|
|
301
|
+
'deleteMissing',
|
|
302
|
+
'transform',
|
|
303
|
+
'idPrefix',
|
|
304
|
+
'idSuffix',
|
|
305
|
+
'webhook',
|
|
306
|
+
'retries',
|
|
307
|
+
'verify',
|
|
308
|
+
'rateLimit',
|
|
309
|
+
'skipOversized',
|
|
310
|
+
'detectConflicts',
|
|
311
|
+
'maxDepth',
|
|
312
|
+
'verifyIntegrity',
|
|
313
|
+
] as const;
|
|
314
|
+
|
|
315
|
+
const merged = {} as Record<string, unknown>;
|
|
316
|
+
for (const key of standardKeys) {
|
|
317
|
+
merged[key] = resolve(key, cliArgs, fileConfig, defaultConfig);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// Special cases: where/rename have non-trivial merge logic
|
|
321
|
+
merged.where =
|
|
322
|
+
cliWhereFilters.length > 0 ? cliWhereFilters : (fileConfig.where ?? defaultConfig.where);
|
|
323
|
+
merged.renameCollection =
|
|
324
|
+
Object.keys(cliRenameCollection).length > 0
|
|
325
|
+
? cliRenameCollection
|
|
326
|
+
: (fileConfig.renameCollection ?? defaultConfig.renameCollection);
|
|
327
|
+
|
|
328
|
+
// CLI-only options (no file config support)
|
|
329
|
+
merged.resume = cliArgs.resume ?? defaultConfig.resume;
|
|
330
|
+
merged.stateFile = cliArgs.stateFile ?? defaultConfig.stateFile;
|
|
331
|
+
merged.json = cliArgs.json ?? defaultConfig.json;
|
|
332
|
+
merged.transformSamples = cliArgs.transformSamples ?? defaultConfig.transformSamples;
|
|
333
|
+
merged.allowHttpWebhook = cliArgs.allowHttpWebhook ?? defaultConfig.allowHttpWebhook;
|
|
334
|
+
|
|
335
|
+
return merged as unknown as Config;
|
|
265
336
|
}
|
package/src/config/validator.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Config, ValidatedConfig } from '../types.js';
|
|
2
|
+
import { MAX_BATCH_SIZE, MAX_PARALLEL, MAX_DEPTH, MAX_RETRIES } from '../constants.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Validate a Firestore collection or document ID.
|
|
@@ -77,6 +78,56 @@ export function validateConfig(config: Config): string[] {
|
|
|
77
78
|
errors.push(...pathErrors);
|
|
78
79
|
}
|
|
79
80
|
|
|
81
|
+
// Validate rename mapping names as valid Firestore IDs
|
|
82
|
+
for (const [source, dest] of Object.entries(config.renameCollection)) {
|
|
83
|
+
const sourceError = validateFirestoreId(source, 'collection');
|
|
84
|
+
if (sourceError) {
|
|
85
|
+
errors.push(`Invalid rename source "${source}": ${sourceError}`);
|
|
86
|
+
}
|
|
87
|
+
const destError = validateFirestoreId(dest, 'collection');
|
|
88
|
+
if (destError) {
|
|
89
|
+
errors.push(`Invalid rename destination "${dest}": ${destError}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Validate numeric bounds
|
|
94
|
+
if (config.batchSize < 1 || config.batchSize > MAX_BATCH_SIZE) {
|
|
95
|
+
errors.push(`Batch size must be between 1 and ${MAX_BATCH_SIZE} (Firestore limit)`);
|
|
96
|
+
}
|
|
97
|
+
if (config.parallel < 1 || config.parallel > MAX_PARALLEL) {
|
|
98
|
+
errors.push(`Parallel must be between 1 and ${MAX_PARALLEL}`);
|
|
99
|
+
}
|
|
100
|
+
if (config.rateLimit < 0) {
|
|
101
|
+
errors.push('Rate limit must be 0 (unlimited) or a positive number');
|
|
102
|
+
}
|
|
103
|
+
if (config.maxDepth < 0 || config.maxDepth > MAX_DEPTH) {
|
|
104
|
+
errors.push(`Max depth must be between 0 (unlimited) and ${MAX_DEPTH}`);
|
|
105
|
+
}
|
|
106
|
+
if (config.retries < 0 || config.retries > MAX_RETRIES) {
|
|
107
|
+
errors.push(`Retries must be between 0 and ${MAX_RETRIES}`);
|
|
108
|
+
}
|
|
109
|
+
if (config.limit < 0) {
|
|
110
|
+
errors.push('Document limit must be 0 (no limit) or a positive number');
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Validate rename mapping for collisions (multiple sources mapping to same dest)
|
|
114
|
+
const renameDestinations = Object.values(config.renameCollection);
|
|
115
|
+
const seenDests = new Set<string>();
|
|
116
|
+
for (const dest of renameDestinations) {
|
|
117
|
+
if (seenDests.has(dest)) {
|
|
118
|
+
errors.push(`Rename collision: multiple collections map to destination "${dest}"`);
|
|
119
|
+
}
|
|
120
|
+
seenDests.add(dest);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Validate idPrefix/idSuffix format
|
|
124
|
+
if (config.idPrefix !== null && config.idPrefix.includes('/')) {
|
|
125
|
+
errors.push('ID prefix cannot contain "/" (would create invalid document paths)');
|
|
126
|
+
}
|
|
127
|
+
if (config.idSuffix !== null && config.idSuffix.includes('/')) {
|
|
128
|
+
errors.push('ID suffix cannot contain "/" (would create invalid document paths)');
|
|
129
|
+
}
|
|
130
|
+
|
|
80
131
|
return errors;
|
|
81
132
|
}
|
|
82
133
|
|
package/src/constants.ts
CHANGED
|
@@ -28,3 +28,42 @@ export const STATE_SAVE_INTERVAL_MS = 5000;
|
|
|
28
28
|
|
|
29
29
|
/** Default number of batches between state saves */
|
|
30
30
|
export const STATE_SAVE_BATCH_INTERVAL = 10;
|
|
31
|
+
|
|
32
|
+
// =============================================================================
|
|
33
|
+
// Validation Limits
|
|
34
|
+
// =============================================================================
|
|
35
|
+
|
|
36
|
+
/** Maximum Firestore batch size */
|
|
37
|
+
export const MAX_BATCH_SIZE = 500;
|
|
38
|
+
|
|
39
|
+
/** Maximum parallel transfers */
|
|
40
|
+
export const MAX_PARALLEL = 20;
|
|
41
|
+
|
|
42
|
+
/** Maximum subcollection recursion depth */
|
|
43
|
+
export const MAX_DEPTH = 100;
|
|
44
|
+
|
|
45
|
+
/** Maximum retry attempts */
|
|
46
|
+
export const MAX_RETRIES = 10;
|
|
47
|
+
|
|
48
|
+
// =============================================================================
|
|
49
|
+
// Webhook Constants
|
|
50
|
+
// =============================================================================
|
|
51
|
+
|
|
52
|
+
/** Webhook request timeout in milliseconds */
|
|
53
|
+
export const WEBHOOK_TIMEOUT_MS = 30_000;
|
|
54
|
+
|
|
55
|
+
/** Maximum webhook payload size in bytes (Discord limit) */
|
|
56
|
+
export const WEBHOOK_MAX_PAYLOAD_BYTES = 2_000_000;
|
|
57
|
+
|
|
58
|
+
/** Maximum webhook retry attempts */
|
|
59
|
+
export const WEBHOOK_MAX_RETRIES = 2;
|
|
60
|
+
|
|
61
|
+
/** Webhook retry base delay in milliseconds */
|
|
62
|
+
export const WEBHOOK_RETRY_DELAY_MS = 1_000;
|
|
63
|
+
|
|
64
|
+
// =============================================================================
|
|
65
|
+
// Clear Collection Constants
|
|
66
|
+
// =============================================================================
|
|
67
|
+
|
|
68
|
+
/** Page size for paginated collection clearing */
|
|
69
|
+
export const CLEAR_PAGE_SIZE = 500;
|
package/src/firebase/index.ts
CHANGED
|
@@ -12,22 +12,21 @@ export interface FirebaseConnections {
|
|
|
12
12
|
destDb: Firestore;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
function getAppOptions(projectId: string): admin.AppOptions {
|
|
16
|
+
const isEmulator = !!process.env.FIRESTORE_EMULATOR_HOST;
|
|
17
|
+
return {
|
|
18
|
+
projectId,
|
|
19
|
+
...(isEmulator ? {} : { credential: admin.credential.applicationDefault() }),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
15
23
|
export function initializeFirebase(config: Config): FirebaseConnections {
|
|
16
|
-
sourceApp
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
projectId: config.sourceProject!,
|
|
20
|
-
},
|
|
21
|
-
'source'
|
|
22
|
-
);
|
|
24
|
+
if (sourceApp || destApp) {
|
|
25
|
+
throw new Error('Firebase already initialized. Call cleanupFirebase() first.');
|
|
26
|
+
}
|
|
23
27
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
credential: admin.credential.applicationDefault(),
|
|
27
|
-
projectId: config.destProject!,
|
|
28
|
-
},
|
|
29
|
-
'dest'
|
|
30
|
-
);
|
|
28
|
+
sourceApp = admin.initializeApp(getAppOptions(config.sourceProject!), 'source');
|
|
29
|
+
destApp = admin.initializeApp(getAppOptions(config.destProject!), 'dest');
|
|
31
30
|
|
|
32
31
|
return {
|
|
33
32
|
sourceDb: sourceApp.firestore(),
|
|
@@ -52,7 +51,8 @@ export async function checkDatabaseConnectivity(
|
|
|
52
51
|
const errorInfo = formatFirebaseError(err);
|
|
53
52
|
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
54
53
|
throw new Error(
|
|
55
|
-
`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}
|
|
54
|
+
`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}`,
|
|
55
|
+
{ cause: error }
|
|
56
56
|
);
|
|
57
57
|
}
|
|
58
58
|
|
|
@@ -66,7 +66,8 @@ export async function checkDatabaseConnectivity(
|
|
|
66
66
|
const errorInfo = formatFirebaseError(err);
|
|
67
67
|
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
68
68
|
throw new Error(
|
|
69
|
-
`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}
|
|
69
|
+
`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}`,
|
|
70
|
+
{ cause: error }
|
|
70
71
|
);
|
|
71
72
|
}
|
|
72
73
|
} else {
|
|
@@ -77,6 +78,16 @@ export async function checkDatabaseConnectivity(
|
|
|
77
78
|
}
|
|
78
79
|
|
|
79
80
|
export async function cleanupFirebase(): Promise<void> {
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
try {
|
|
82
|
+
if (sourceApp) await sourceApp.delete();
|
|
83
|
+
} catch {
|
|
84
|
+
// Ignore cleanup errors - app may already be deleted
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
if (destApp) await destApp.delete();
|
|
88
|
+
} catch {
|
|
89
|
+
// Ignore cleanup errors - app may already be deleted
|
|
90
|
+
}
|
|
91
|
+
sourceApp = null;
|
|
92
|
+
destApp = null;
|
|
82
93
|
}
|