@fazetitans/fscopy 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -1
- package/package.json +19 -15
- package/src/cli.ts +10 -2
- package/src/config/defaults.ts +1 -0
- package/src/config/parser.ts +90 -53
- package/src/config/validator.ts +51 -0
- package/src/constants.ts +39 -0
- package/src/firebase/index.ts +29 -18
- package/src/interactive.ts +6 -1
- package/src/orchestrator.ts +69 -57
- package/src/state/index.ts +42 -26
- package/src/transfer/clear.ts +104 -68
- package/src/transfer/count.ts +44 -35
- package/src/transfer/helpers.ts +36 -1
- package/src/transfer/index.ts +7 -1
- package/src/transfer/transfer.ts +141 -149
- package/src/transform/loader.ts +13 -1
- package/src/types.ts +3 -1
- package/src/utils/credentials.ts +7 -4
- package/src/utils/errors.ts +7 -2
- package/src/utils/index.ts +6 -1
- package/src/utils/output.ts +6 -0
- package/src/utils/progress.ts +10 -0
- package/src/webhook/index.ts +127 -44
package/README.md
CHANGED
|
@@ -389,6 +389,7 @@ fscopy --init config.json
|
|
|
389
389
|
| `--max-depth` | | number | `0` | Max subcollection depth (0 = unlimited) |
|
|
390
390
|
| `--detect-conflicts` | | boolean | `false` | Detect concurrent modifications |
|
|
391
391
|
| `--verify-integrity` | | boolean | `false` | Verify document integrity with hash |
|
|
392
|
+
| `--allow-http-webhook` | | boolean | `false` | Allow HTTP (non-HTTPS) webhook URLs |
|
|
392
393
|
|
|
393
394
|
## How It Works
|
|
394
395
|
|
|
@@ -402,7 +403,9 @@ fscopy --init config.json
|
|
|
402
403
|
|
|
403
404
|
- **Transform files execute arbitrary code** - The `--transform` option uses dynamic imports to load and execute JavaScript/TypeScript files. Only use transform files you have written or thoroughly reviewed. Malicious transform files could access your filesystem, network, or credentials.
|
|
404
405
|
|
|
405
|
-
- **Webhook URLs
|
|
406
|
+
- **Webhook URLs must use HTTPS** - fscopy blocks HTTP webhooks by default (except localhost). Use `--allow-http-webhook` to explicitly allow unencrypted webhooks. Webhook payloads contain project names and transfer statistics that could be sensitive.
|
|
407
|
+
|
|
408
|
+
- **Transform file extensions are validated** - Only `.ts`, `.js`, `.mjs`, and `.mts` files are accepted as transform files.
|
|
406
409
|
|
|
407
410
|
- **Credentials via ADC** - fscopy uses Google Application Default Credentials. Ensure you're authenticated with the correct account before running transfers.
|
|
408
411
|
|
|
@@ -416,8 +419,16 @@ fscopy --init config.json
|
|
|
416
419
|
- **Clear is destructive** - `--clear` deletes all destination docs before transfer
|
|
417
420
|
- **Delete-missing syncs** - `--delete-missing` removes orphan docs after transfer
|
|
418
421
|
- **Transform applies to all** - Transform function is applied to both root and subcollection docs
|
|
422
|
+
- **Limit applies to root only** - `--limit` restricts the number of documents at the root collection level; subcollections are always copied in full
|
|
419
423
|
- **Same project allowed** - Source and destination can be the same project when using `--rename-collection` or `--id-prefix`/`--id-suffix`
|
|
420
424
|
|
|
425
|
+
## Environment Variables
|
|
426
|
+
|
|
427
|
+
| Variable | Description |
|
|
428
|
+
| -------- | ----------- |
|
|
429
|
+
| `GOOGLE_APPLICATION_CREDENTIALS` | Path to Google Cloud credentials JSON file |
|
|
430
|
+
| `FSCOPY_SKIP_CREDENTIALS_CHECK` | Set to `1` to skip the credentials check at startup (useful in CI/CD with alternative auth) |
|
|
431
|
+
|
|
421
432
|
## Limitations
|
|
422
433
|
|
|
423
434
|
### Firestore Special Types
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fazetitans/fscopy",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Fast CLI tool to copy Firestore collections between Firebase projects with filtering, parallel transfers, and subcollection support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -9,15 +9,18 @@
|
|
|
9
9
|
"scripts": {
|
|
10
10
|
"start": "bun src/cli.ts",
|
|
11
11
|
"dev": "bun --watch src/cli.ts",
|
|
12
|
-
"test": "bun test",
|
|
12
|
+
"test": "bun test src/__tests__/*.test.ts",
|
|
13
|
+
"test:integration": "firebase emulators:exec --only firestore --project fscopy-test 'FIRESTORE_EMULATOR_HOST=127.0.0.1:8080 FSCOPY_SKIP_CREDENTIALS_CHECK=1 bun test src/__tests__/integration/'",
|
|
14
|
+
"test:all": "bun run test && bun run test:integration",
|
|
13
15
|
"test:watch": "bun test --watch",
|
|
14
|
-
"test:coverage": "bun test --coverage",
|
|
16
|
+
"test:coverage": "bun test --coverage src/__tests__/*.test.ts",
|
|
17
|
+
"emulator": "firebase emulators:start --only firestore --project fscopy-test",
|
|
15
18
|
"type-check": "tsc --noEmit",
|
|
16
19
|
"lint": "eslint src/**/*.ts --no-warn-ignored",
|
|
17
20
|
"lint:fix": "eslint src/**/*.ts --no-warn-ignored --fix",
|
|
18
21
|
"format": "prettier --write src/**/*.ts",
|
|
19
22
|
"format:check": "prettier --check src/**/*.ts",
|
|
20
|
-
"prepublishOnly": "bun run type-check && bun run lint && bun test"
|
|
23
|
+
"prepublishOnly": "bun run type-check && bun run lint && bun run test"
|
|
21
24
|
},
|
|
22
25
|
"keywords": [
|
|
23
26
|
"firestore",
|
|
@@ -54,24 +57,25 @@
|
|
|
54
57
|
"LICENSE"
|
|
55
58
|
],
|
|
56
59
|
"dependencies": {
|
|
57
|
-
"@inquirer/prompts": "^8.
|
|
60
|
+
"@inquirer/prompts": "^8.3.0",
|
|
58
61
|
"cli-progress": "^3.12.0",
|
|
59
|
-
"firebase-admin": "^13.
|
|
62
|
+
"firebase-admin": "^13.7.0",
|
|
60
63
|
"ini": "^6.0.0",
|
|
61
|
-
"yargs": "^
|
|
64
|
+
"yargs": "^18.0.0"
|
|
62
65
|
},
|
|
63
66
|
"devDependencies": {
|
|
64
|
-
"@eslint/js": "^
|
|
67
|
+
"@eslint/js": "^10.0.1",
|
|
65
68
|
"@types/cli-progress": "^3.11.6",
|
|
66
69
|
"@types/ini": "^4.1.1",
|
|
67
|
-
"@types/node": "^25.
|
|
70
|
+
"@types/node": "^25.3.5",
|
|
68
71
|
"@types/yargs": "^17.0.35",
|
|
69
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
70
|
-
"@typescript-eslint/parser": "^8.
|
|
71
|
-
"bun-types": "^1.3.
|
|
72
|
-
"eslint": "^
|
|
73
|
-
"prettier": "^3.
|
|
72
|
+
"@typescript-eslint/eslint-plugin": "^8.56.1",
|
|
73
|
+
"@typescript-eslint/parser": "^8.56.1",
|
|
74
|
+
"bun-types": "^1.3.10",
|
|
75
|
+
"eslint": "^10.0.2",
|
|
76
|
+
"prettier": "^3.8.1",
|
|
74
77
|
"typescript": "^5.9.3",
|
|
75
|
-
"
|
|
78
|
+
"firebase-tools": "^15.9.0",
|
|
79
|
+
"typescript-eslint": "^8.56.1"
|
|
76
80
|
}
|
|
77
81
|
}
|
package/src/cli.ts
CHANGED
|
@@ -61,7 +61,7 @@ const argv = yargs(hideBin(process.argv))
|
|
|
61
61
|
.option('limit', {
|
|
62
62
|
alias: 'l',
|
|
63
63
|
type: 'number',
|
|
64
|
-
description: 'Limit number of documents per collection (0 = no limit)',
|
|
64
|
+
description: 'Limit number of documents per root collection (0 = no limit, does not apply to subcollections)',
|
|
65
65
|
})
|
|
66
66
|
.option('source-project', {
|
|
67
67
|
type: 'string',
|
|
@@ -196,6 +196,11 @@ const argv = yargs(hideBin(process.argv))
|
|
|
196
196
|
type: 'boolean',
|
|
197
197
|
description: 'Verify document integrity with hash after transfer',
|
|
198
198
|
})
|
|
199
|
+
.option('allow-http-webhook', {
|
|
200
|
+
type: 'boolean',
|
|
201
|
+
description: 'Allow webhook URLs using HTTP instead of HTTPS',
|
|
202
|
+
default: false,
|
|
203
|
+
})
|
|
199
204
|
.option('validate-only', {
|
|
200
205
|
type: 'boolean',
|
|
201
206
|
description: 'Only validate config and display it (no transfer)',
|
|
@@ -269,7 +274,10 @@ async function main(): Promise<void> {
|
|
|
269
274
|
|
|
270
275
|
// Validate webhook URL if configured
|
|
271
276
|
if (validConfig.webhook) {
|
|
272
|
-
const webhookValidation = validateWebhookUrl(
|
|
277
|
+
const webhookValidation = validateWebhookUrl(
|
|
278
|
+
validConfig.webhook,
|
|
279
|
+
validConfig.allowHttpWebhook
|
|
280
|
+
);
|
|
273
281
|
if (!webhookValidation.valid) {
|
|
274
282
|
console.log(`\n❌ ${webhookValidation.warning}`);
|
|
275
283
|
process.exit(1);
|
package/src/config/defaults.ts
CHANGED
package/src/config/parser.ts
CHANGED
|
@@ -3,9 +3,18 @@ import path from 'node:path';
|
|
|
3
3
|
import ini from 'ini';
|
|
4
4
|
import type { Config, WhereFilter, CliArgs } from '../types.js';
|
|
5
5
|
|
|
6
|
+
const SUPPORTED_EXTENSIONS = new Set(['.ini', '.json', '.cfg', '.conf']);
|
|
7
|
+
|
|
8
|
+
function stderrWarn(message: string): void {
|
|
9
|
+
process.stderr.write(`${message}\n`);
|
|
10
|
+
}
|
|
11
|
+
|
|
6
12
|
export function getFileFormat(filePath: string): 'json' | 'ini' {
|
|
7
13
|
const ext = path.extname(filePath).toLowerCase();
|
|
8
14
|
if (ext === '.json') return 'json';
|
|
15
|
+
if (ext && !SUPPORTED_EXTENSIONS.has(ext)) {
|
|
16
|
+
stderrWarn(`⚠️ Unrecognized config file extension "${ext}", parsing as INI format`);
|
|
17
|
+
}
|
|
9
18
|
return 'ini';
|
|
10
19
|
}
|
|
11
20
|
|
|
@@ -22,7 +31,7 @@ export function parseWhereFilter(filterStr: string): WhereFilter | null {
|
|
|
22
31
|
const match = new RegExp(operatorRegex).exec(filterStr);
|
|
23
32
|
|
|
24
33
|
if (!match) {
|
|
25
|
-
|
|
34
|
+
stderrWarn(`⚠️ Invalid where filter: "${filterStr}" (missing operator)`);
|
|
26
35
|
return null;
|
|
27
36
|
}
|
|
28
37
|
|
|
@@ -30,20 +39,20 @@ export function parseWhereFilter(filterStr: string): WhereFilter | null {
|
|
|
30
39
|
const [fieldPart, valuePart] = filterStr.split(operatorRegex).filter((_, i) => i !== 1);
|
|
31
40
|
|
|
32
41
|
if (!fieldPart || !valuePart) {
|
|
33
|
-
|
|
42
|
+
stderrWarn(`⚠️ Invalid where filter: "${filterStr}" (missing field or value)`);
|
|
34
43
|
return null;
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
const field = fieldPart.trim();
|
|
38
47
|
const rawValue = valuePart.trim();
|
|
39
48
|
|
|
40
|
-
let value: string | number | boolean;
|
|
49
|
+
let value: string | number | boolean | null;
|
|
41
50
|
if (rawValue === 'true') {
|
|
42
51
|
value = true;
|
|
43
52
|
} else if (rawValue === 'false') {
|
|
44
53
|
value = false;
|
|
45
54
|
} else if (rawValue === 'null') {
|
|
46
|
-
value = null
|
|
55
|
+
value = null;
|
|
47
56
|
} else if (!Number.isNaN(Number(rawValue)) && rawValue !== '') {
|
|
48
57
|
value = Number(rawValue);
|
|
49
58
|
} else {
|
|
@@ -84,13 +93,13 @@ export function parseRenameMapping(
|
|
|
84
93
|
const mapping = String(item).trim();
|
|
85
94
|
const colonIndex = mapping.indexOf(':');
|
|
86
95
|
if (colonIndex === -1) {
|
|
87
|
-
|
|
96
|
+
stderrWarn(`⚠️ Invalid rename mapping: "${mapping}" (missing ':')`);
|
|
88
97
|
continue;
|
|
89
98
|
}
|
|
90
99
|
const source = mapping.slice(0, colonIndex).trim();
|
|
91
100
|
const dest = mapping.slice(colonIndex + 1).trim();
|
|
92
101
|
if (!source || !dest) {
|
|
93
|
-
|
|
102
|
+
stderrWarn(`⚠️ Invalid rename mapping: "${mapping}" (empty source or dest)`);
|
|
94
103
|
continue;
|
|
95
104
|
}
|
|
96
105
|
result[source] = dest;
|
|
@@ -164,11 +173,21 @@ export function parseIniConfig(content: string): Partial<Config> {
|
|
|
164
173
|
webhook: parsed.options?.webhook ?? null,
|
|
165
174
|
retries: parseIntOption(parsed.options?.retries),
|
|
166
175
|
rateLimit: parseIntOption(parsed.options?.rateLimit),
|
|
167
|
-
skipOversized:
|
|
168
|
-
|
|
176
|
+
skipOversized:
|
|
177
|
+
parsed.options?.skipOversized !== undefined
|
|
178
|
+
? parseBoolean(parsed.options.skipOversized)
|
|
179
|
+
: undefined,
|
|
180
|
+
detectConflicts:
|
|
181
|
+
parsed.options?.detectConflicts !== undefined
|
|
182
|
+
? parseBoolean(parsed.options.detectConflicts)
|
|
183
|
+
: undefined,
|
|
169
184
|
maxDepth: parseIntOption(parsed.options?.maxDepth),
|
|
170
|
-
verify:
|
|
171
|
-
|
|
185
|
+
verify:
|
|
186
|
+
parsed.options?.verify !== undefined ? parseBoolean(parsed.options.verify) : undefined,
|
|
187
|
+
verifyIntegrity:
|
|
188
|
+
parsed.options?.verifyIntegrity !== undefined
|
|
189
|
+
? parseBoolean(parsed.options.verifyIntegrity)
|
|
190
|
+
: undefined,
|
|
172
191
|
};
|
|
173
192
|
}
|
|
174
193
|
|
|
@@ -241,11 +260,23 @@ export function loadConfigFile(configPath?: string): Partial<Config> {
|
|
|
241
260
|
const content = fs.readFileSync(absolutePath, 'utf-8');
|
|
242
261
|
const format = getFileFormat(absolutePath);
|
|
243
262
|
|
|
244
|
-
|
|
263
|
+
process.stderr.write(`📄 Loaded config from: ${absolutePath} (${format.toUpperCase()})\n`);
|
|
245
264
|
|
|
246
265
|
return format === 'json' ? parseJsonConfig(content) : parseIniConfig(content);
|
|
247
266
|
}
|
|
248
267
|
|
|
268
|
+
/**
|
|
269
|
+
* Resolve a config value with priority: CLI > file > default.
|
|
270
|
+
*/
|
|
271
|
+
function resolve<K extends keyof Config>(
|
|
272
|
+
key: K,
|
|
273
|
+
cliArgs: Partial<Record<K, Config[K]>>,
|
|
274
|
+
fileConfig: Partial<Config>,
|
|
275
|
+
defaultConfig: Config
|
|
276
|
+
): Config[K] {
|
|
277
|
+
return (cliArgs[key] ?? fileConfig[key] ?? defaultConfig[key]) as Config[K];
|
|
278
|
+
}
|
|
279
|
+
|
|
249
280
|
export function mergeConfig(
|
|
250
281
|
defaultConfig: Config,
|
|
251
282
|
fileConfig: Partial<Config>,
|
|
@@ -254,46 +285,52 @@ export function mergeConfig(
|
|
|
254
285
|
const cliWhereFilters = parseWhereFilters(cliArgs.where);
|
|
255
286
|
const cliRenameCollection = parseRenameMapping(cliArgs.renameCollection);
|
|
256
287
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
288
|
+
// Keys that follow standard CLI > file > default priority
|
|
289
|
+
const standardKeys = [
|
|
290
|
+
'collections',
|
|
291
|
+
'includeSubcollections',
|
|
292
|
+
'dryRun',
|
|
293
|
+
'batchSize',
|
|
294
|
+
'limit',
|
|
295
|
+
'sourceProject',
|
|
296
|
+
'destProject',
|
|
297
|
+
'exclude',
|
|
298
|
+
'merge',
|
|
299
|
+
'parallel',
|
|
300
|
+
'clear',
|
|
301
|
+
'deleteMissing',
|
|
302
|
+
'transform',
|
|
303
|
+
'idPrefix',
|
|
304
|
+
'idSuffix',
|
|
305
|
+
'webhook',
|
|
306
|
+
'retries',
|
|
307
|
+
'verify',
|
|
308
|
+
'rateLimit',
|
|
309
|
+
'skipOversized',
|
|
310
|
+
'detectConflicts',
|
|
311
|
+
'maxDepth',
|
|
312
|
+
'verifyIntegrity',
|
|
313
|
+
] as const;
|
|
314
|
+
|
|
315
|
+
const merged = {} as Record<string, unknown>;
|
|
316
|
+
for (const key of standardKeys) {
|
|
317
|
+
merged[key] = resolve(key, cliArgs, fileConfig, defaultConfig);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// Special cases: where/rename have non-trivial merge logic
|
|
321
|
+
merged.where =
|
|
322
|
+
cliWhereFilters.length > 0 ? cliWhereFilters : (fileConfig.where ?? defaultConfig.where);
|
|
323
|
+
merged.renameCollection =
|
|
324
|
+
Object.keys(cliRenameCollection).length > 0
|
|
325
|
+
? cliRenameCollection
|
|
326
|
+
: (fileConfig.renameCollection ?? defaultConfig.renameCollection);
|
|
327
|
+
|
|
328
|
+
// CLI-only options (no file config support)
|
|
329
|
+
merged.resume = cliArgs.resume ?? defaultConfig.resume;
|
|
330
|
+
merged.stateFile = cliArgs.stateFile ?? defaultConfig.stateFile;
|
|
331
|
+
merged.json = cliArgs.json ?? defaultConfig.json;
|
|
332
|
+
merged.transformSamples = cliArgs.transformSamples ?? defaultConfig.transformSamples;
|
|
333
|
+
merged.allowHttpWebhook = cliArgs.allowHttpWebhook ?? defaultConfig.allowHttpWebhook;
|
|
334
|
+
|
|
335
|
+
return merged as unknown as Config;
|
|
299
336
|
}
|
package/src/config/validator.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Config, ValidatedConfig } from '../types.js';
|
|
2
|
+
import { MAX_BATCH_SIZE, MAX_PARALLEL, MAX_DEPTH, MAX_RETRIES } from '../constants.js';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Validate a Firestore collection or document ID.
|
|
@@ -77,6 +78,56 @@ export function validateConfig(config: Config): string[] {
|
|
|
77
78
|
errors.push(...pathErrors);
|
|
78
79
|
}
|
|
79
80
|
|
|
81
|
+
// Validate rename mapping names as valid Firestore IDs
|
|
82
|
+
for (const [source, dest] of Object.entries(config.renameCollection)) {
|
|
83
|
+
const sourceError = validateFirestoreId(source, 'collection');
|
|
84
|
+
if (sourceError) {
|
|
85
|
+
errors.push(`Invalid rename source "${source}": ${sourceError}`);
|
|
86
|
+
}
|
|
87
|
+
const destError = validateFirestoreId(dest, 'collection');
|
|
88
|
+
if (destError) {
|
|
89
|
+
errors.push(`Invalid rename destination "${dest}": ${destError}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Validate numeric bounds
|
|
94
|
+
if (config.batchSize < 1 || config.batchSize > MAX_BATCH_SIZE) {
|
|
95
|
+
errors.push(`Batch size must be between 1 and ${MAX_BATCH_SIZE} (Firestore limit)`);
|
|
96
|
+
}
|
|
97
|
+
if (config.parallel < 1 || config.parallel > MAX_PARALLEL) {
|
|
98
|
+
errors.push(`Parallel must be between 1 and ${MAX_PARALLEL}`);
|
|
99
|
+
}
|
|
100
|
+
if (config.rateLimit < 0) {
|
|
101
|
+
errors.push('Rate limit must be 0 (unlimited) or a positive number');
|
|
102
|
+
}
|
|
103
|
+
if (config.maxDepth < 0 || config.maxDepth > MAX_DEPTH) {
|
|
104
|
+
errors.push(`Max depth must be between 0 (unlimited) and ${MAX_DEPTH}`);
|
|
105
|
+
}
|
|
106
|
+
if (config.retries < 0 || config.retries > MAX_RETRIES) {
|
|
107
|
+
errors.push(`Retries must be between 0 and ${MAX_RETRIES}`);
|
|
108
|
+
}
|
|
109
|
+
if (config.limit < 0) {
|
|
110
|
+
errors.push('Document limit must be 0 (no limit) or a positive number');
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Validate rename mapping for collisions (multiple sources mapping to same dest)
|
|
114
|
+
const renameDestinations = Object.values(config.renameCollection);
|
|
115
|
+
const seenDests = new Set<string>();
|
|
116
|
+
for (const dest of renameDestinations) {
|
|
117
|
+
if (seenDests.has(dest)) {
|
|
118
|
+
errors.push(`Rename collision: multiple collections map to destination "${dest}"`);
|
|
119
|
+
}
|
|
120
|
+
seenDests.add(dest);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Validate idPrefix/idSuffix format
|
|
124
|
+
if (config.idPrefix !== null && config.idPrefix.includes('/')) {
|
|
125
|
+
errors.push('ID prefix cannot contain "/" (would create invalid document paths)');
|
|
126
|
+
}
|
|
127
|
+
if (config.idSuffix !== null && config.idSuffix.includes('/')) {
|
|
128
|
+
errors.push('ID suffix cannot contain "/" (would create invalid document paths)');
|
|
129
|
+
}
|
|
130
|
+
|
|
80
131
|
return errors;
|
|
81
132
|
}
|
|
82
133
|
|
package/src/constants.ts
CHANGED
|
@@ -28,3 +28,42 @@ export const STATE_SAVE_INTERVAL_MS = 5000;
|
|
|
28
28
|
|
|
29
29
|
/** Default number of batches between state saves */
|
|
30
30
|
export const STATE_SAVE_BATCH_INTERVAL = 10;
|
|
31
|
+
|
|
32
|
+
// =============================================================================
|
|
33
|
+
// Validation Limits
|
|
34
|
+
// =============================================================================
|
|
35
|
+
|
|
36
|
+
/** Maximum Firestore batch size */
|
|
37
|
+
export const MAX_BATCH_SIZE = 500;
|
|
38
|
+
|
|
39
|
+
/** Maximum parallel transfers */
|
|
40
|
+
export const MAX_PARALLEL = 20;
|
|
41
|
+
|
|
42
|
+
/** Maximum subcollection recursion depth */
|
|
43
|
+
export const MAX_DEPTH = 100;
|
|
44
|
+
|
|
45
|
+
/** Maximum retry attempts */
|
|
46
|
+
export const MAX_RETRIES = 10;
|
|
47
|
+
|
|
48
|
+
// =============================================================================
|
|
49
|
+
// Webhook Constants
|
|
50
|
+
// =============================================================================
|
|
51
|
+
|
|
52
|
+
/** Webhook request timeout in milliseconds */
|
|
53
|
+
export const WEBHOOK_TIMEOUT_MS = 30_000;
|
|
54
|
+
|
|
55
|
+
/** Maximum webhook payload size in bytes (Discord limit) */
|
|
56
|
+
export const WEBHOOK_MAX_PAYLOAD_BYTES = 2_000_000;
|
|
57
|
+
|
|
58
|
+
/** Maximum webhook retry attempts */
|
|
59
|
+
export const WEBHOOK_MAX_RETRIES = 2;
|
|
60
|
+
|
|
61
|
+
/** Webhook retry base delay in milliseconds */
|
|
62
|
+
export const WEBHOOK_RETRY_DELAY_MS = 1_000;
|
|
63
|
+
|
|
64
|
+
// =============================================================================
|
|
65
|
+
// Clear Collection Constants
|
|
66
|
+
// =============================================================================
|
|
67
|
+
|
|
68
|
+
/** Page size for paginated collection clearing */
|
|
69
|
+
export const CLEAR_PAGE_SIZE = 500;
|
package/src/firebase/index.ts
CHANGED
|
@@ -12,22 +12,21 @@ export interface FirebaseConnections {
|
|
|
12
12
|
destDb: Firestore;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
function getAppOptions(projectId: string): admin.AppOptions {
|
|
16
|
+
const isEmulator = !!process.env.FIRESTORE_EMULATOR_HOST;
|
|
17
|
+
return {
|
|
18
|
+
projectId,
|
|
19
|
+
...(isEmulator ? {} : { credential: admin.credential.applicationDefault() }),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
15
23
|
export function initializeFirebase(config: Config): FirebaseConnections {
|
|
16
|
-
sourceApp
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
projectId: config.sourceProject!,
|
|
20
|
-
},
|
|
21
|
-
'source'
|
|
22
|
-
);
|
|
24
|
+
if (sourceApp || destApp) {
|
|
25
|
+
throw new Error('Firebase already initialized. Call cleanupFirebase() first.');
|
|
26
|
+
}
|
|
23
27
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
credential: admin.credential.applicationDefault(),
|
|
27
|
-
projectId: config.destProject!,
|
|
28
|
-
},
|
|
29
|
-
'dest'
|
|
30
|
-
);
|
|
28
|
+
sourceApp = admin.initializeApp(getAppOptions(config.sourceProject!), 'source');
|
|
29
|
+
destApp = admin.initializeApp(getAppOptions(config.destProject!), 'dest');
|
|
31
30
|
|
|
32
31
|
return {
|
|
33
32
|
sourceDb: sourceApp.firestore(),
|
|
@@ -52,7 +51,8 @@ export async function checkDatabaseConnectivity(
|
|
|
52
51
|
const errorInfo = formatFirebaseError(err);
|
|
53
52
|
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
54
53
|
throw new Error(
|
|
55
|
-
`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}
|
|
54
|
+
`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}`,
|
|
55
|
+
{ cause: error }
|
|
56
56
|
);
|
|
57
57
|
}
|
|
58
58
|
|
|
@@ -66,7 +66,8 @@ export async function checkDatabaseConnectivity(
|
|
|
66
66
|
const errorInfo = formatFirebaseError(err);
|
|
67
67
|
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
68
68
|
throw new Error(
|
|
69
|
-
`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}
|
|
69
|
+
`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}`,
|
|
70
|
+
{ cause: error }
|
|
70
71
|
);
|
|
71
72
|
}
|
|
72
73
|
} else {
|
|
@@ -77,6 +78,16 @@ export async function checkDatabaseConnectivity(
|
|
|
77
78
|
}
|
|
78
79
|
|
|
79
80
|
export async function cleanupFirebase(): Promise<void> {
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
try {
|
|
82
|
+
if (sourceApp) await sourceApp.delete();
|
|
83
|
+
} catch {
|
|
84
|
+
// Ignore cleanup errors - app may already be deleted
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
if (destApp) await destApp.delete();
|
|
88
|
+
} catch {
|
|
89
|
+
// Ignore cleanup errors - app may already be deleted
|
|
90
|
+
}
|
|
91
|
+
sourceApp = null;
|
|
92
|
+
destApp = null;
|
|
82
93
|
}
|
package/src/interactive.ts
CHANGED
|
@@ -96,7 +96,7 @@ async function discoverCollections(
|
|
|
96
96
|
): Promise<{ app: admin.app.App; db: Firestore; collections: CollectionInfo[] }> {
|
|
97
97
|
console.log('\nConnecting to source project...');
|
|
98
98
|
|
|
99
|
-
let tempSourceApp: admin.app.App;
|
|
99
|
+
let tempSourceApp: admin.app.App | undefined;
|
|
100
100
|
let sourceDb: Firestore;
|
|
101
101
|
let rootCollections: FirebaseFirestore.CollectionReference[];
|
|
102
102
|
|
|
@@ -111,6 +111,11 @@ async function discoverCollections(
|
|
|
111
111
|
sourceDb = tempSourceApp.firestore();
|
|
112
112
|
rootCollections = await sourceDb.listCollections();
|
|
113
113
|
} catch (error) {
|
|
114
|
+
// Clean up Firebase app if it was initialized before the error
|
|
115
|
+
if (tempSourceApp) {
|
|
116
|
+
await tempSourceApp.delete().catch(() => {});
|
|
117
|
+
}
|
|
118
|
+
|
|
114
119
|
const err = error as Error & { code?: string };
|
|
115
120
|
console.error('\nCannot connect to Firebase project:', err.message);
|
|
116
121
|
|