@fazetitans/fscopy 1.1.3 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -33
- package/package.json +3 -3
- package/src/cli.ts +73 -621
- package/src/config/defaults.ts +4 -0
- package/src/config/parser.ts +4 -0
- package/src/config/validator.ts +52 -0
- package/src/firebase/index.ts +82 -0
- package/src/interactive.ts +59 -56
- package/src/orchestrator.ts +408 -0
- package/src/output/display.ts +226 -0
- package/src/state/index.ts +188 -1
- package/src/transfer/clear.ts +162 -104
- package/src/transfer/count.ts +94 -44
- package/src/transfer/transfer.ts +487 -156
- package/src/transform/loader.ts +31 -0
- package/src/types.ts +18 -0
- package/src/utils/credentials.ts +9 -4
- package/src/utils/doc-size.ts +41 -70
- package/src/utils/errors.ts +1 -1
- package/src/utils/index.ts +2 -1
- package/src/utils/integrity.ts +122 -0
- package/src/utils/logger.ts +59 -3
- package/src/utils/output.ts +265 -0
- package/src/utils/progress.ts +102 -0
- package/src/utils/rate-limiter.ts +4 -2
- package/src/webhook/index.ts +6 -6
package/src/cli.ts
CHANGED
|
@@ -3,29 +3,21 @@
|
|
|
3
3
|
// Suppress GCE metadata lookup warning (we're not running on Google Cloud)
|
|
4
4
|
process.env.METADATA_SERVER_DETECTION = 'none';
|
|
5
5
|
|
|
6
|
-
import admin from 'firebase-admin';
|
|
7
|
-
import type { Firestore } from 'firebase-admin/firestore';
|
|
8
6
|
import yargs from 'yargs';
|
|
9
7
|
import { hideBin } from 'yargs/helpers';
|
|
10
|
-
import cliProgress from 'cli-progress';
|
|
11
|
-
import fs from 'node:fs';
|
|
12
|
-
import path from 'node:path';
|
|
13
|
-
import readline from 'node:readline';
|
|
14
8
|
|
|
15
|
-
|
|
16
|
-
import type { Config,
|
|
17
|
-
import {
|
|
9
|
+
import pkg from '../package.json';
|
|
10
|
+
import type { Config, CliArgs } from './types.js';
|
|
11
|
+
import { Output, parseSize } from './utils/output.js';
|
|
18
12
|
import { ensureCredentials } from './utils/credentials.js';
|
|
19
|
-
import { formatFirebaseError } from './utils/errors.js';
|
|
20
|
-
import { RateLimiter } from './utils/rate-limiter.js';
|
|
21
13
|
import { loadConfigFile, mergeConfig } from './config/parser.js';
|
|
22
14
|
import { validateConfig } from './config/validator.js';
|
|
23
15
|
import { defaults } from './config/defaults.js';
|
|
24
16
|
import { generateConfigFile } from './config/generator.js';
|
|
25
|
-
import {
|
|
26
|
-
import { sendWebhook, validateWebhookUrl } from './webhook/index.js';
|
|
27
|
-
import { countDocuments, transferCollection, clearCollection, deleteOrphanDocuments, processInParallel, getDestCollectionPath, type TransferContext, type CountProgress } from './transfer/index.js';
|
|
17
|
+
import { validateWebhookUrl } from './webhook/index.js';
|
|
28
18
|
import { runInteractiveMode } from './interactive.js';
|
|
19
|
+
import { displayConfig, askConfirmation } from './output/display.js';
|
|
20
|
+
import { runTransfer } from './orchestrator.js';
|
|
29
21
|
|
|
30
22
|
// =============================================================================
|
|
31
23
|
// CLI Arguments
|
|
@@ -33,6 +25,7 @@ import { runInteractiveMode } from './interactive.js';
|
|
|
33
25
|
|
|
34
26
|
const argv = yargs(hideBin(process.argv))
|
|
35
27
|
.scriptName('fscopy')
|
|
28
|
+
.version(pkg.version)
|
|
36
29
|
.usage('$0 [options]')
|
|
37
30
|
.option('init', {
|
|
38
31
|
type: 'string',
|
|
@@ -88,6 +81,11 @@ const argv = yargs(hideBin(process.argv))
|
|
|
88
81
|
type: 'string',
|
|
89
82
|
description: 'Path to log file for transfer details',
|
|
90
83
|
})
|
|
84
|
+
.option('max-log-size', {
|
|
85
|
+
type: 'string',
|
|
86
|
+
description: 'Max log file size before rotation (e.g., "10MB", "1GB"). 0 = no rotation.',
|
|
87
|
+
default: '0',
|
|
88
|
+
})
|
|
91
89
|
.option('retries', {
|
|
92
90
|
type: 'number',
|
|
93
91
|
description: 'Number of retries on error (default: 3)',
|
|
@@ -189,6 +187,32 @@ const argv = yargs(hideBin(process.argv))
|
|
|
189
187
|
description: 'Output results in JSON format (for CI/CD)',
|
|
190
188
|
default: false,
|
|
191
189
|
})
|
|
190
|
+
.option('transform-samples', {
|
|
191
|
+
type: 'number',
|
|
192
|
+
description: 'Number of documents to test per collection during transform validation (0 = skip, -1 = all)',
|
|
193
|
+
default: 3,
|
|
194
|
+
})
|
|
195
|
+
.option('detect-conflicts', {
|
|
196
|
+
type: 'boolean',
|
|
197
|
+
description: 'Detect if destination docs were modified during transfer',
|
|
198
|
+
default: false,
|
|
199
|
+
})
|
|
200
|
+
.option('max-depth', {
|
|
201
|
+
type: 'number',
|
|
202
|
+
description: 'Max subcollection depth (0 = unlimited)',
|
|
203
|
+
default: 0,
|
|
204
|
+
})
|
|
205
|
+
.option('verify-integrity', {
|
|
206
|
+
type: 'boolean',
|
|
207
|
+
description: 'Verify document integrity with hash after transfer',
|
|
208
|
+
default: false,
|
|
209
|
+
})
|
|
210
|
+
.option('validate-only', {
|
|
211
|
+
type: 'boolean',
|
|
212
|
+
description: 'Only validate config and display it (no transfer)',
|
|
213
|
+
default: false,
|
|
214
|
+
hidden: true,
|
|
215
|
+
})
|
|
192
216
|
.example('$0 --init config.ini', 'Generate INI config template (default)')
|
|
193
217
|
.example('$0 --init config.json', 'Generate JSON config template')
|
|
194
218
|
.example('$0 -f config.ini', 'Run transfer with config file')
|
|
@@ -209,224 +233,24 @@ const argv = yargs(hideBin(process.argv))
|
|
|
209
233
|
.help()
|
|
210
234
|
.parseSync() as CliArgs;
|
|
211
235
|
|
|
212
|
-
|
|
213
|
-
// =============================================================================
|
|
214
|
-
// Transform Loading
|
|
215
|
-
// =============================================================================
|
|
216
|
-
|
|
217
|
-
async function loadTransformFunction(transformPath: string): Promise<TransformFunction> {
|
|
218
|
-
const absolutePath = path.resolve(transformPath);
|
|
219
|
-
|
|
220
|
-
if (!fs.existsSync(absolutePath)) {
|
|
221
|
-
throw new Error(`Transform file not found: ${absolutePath}`);
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
try {
|
|
225
|
-
const module = await import(absolutePath);
|
|
226
|
-
|
|
227
|
-
// Look for 'transform' export (default or named)
|
|
228
|
-
const transformFn = module.default?.transform ?? module.transform ?? module.default;
|
|
229
|
-
|
|
230
|
-
if (typeof transformFn !== 'function') {
|
|
231
|
-
throw new Error(
|
|
232
|
-
`Transform file must export a 'transform' function. Got: ${typeof transformFn}`
|
|
233
|
-
);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
return transformFn as TransformFunction;
|
|
237
|
-
} catch (error) {
|
|
238
|
-
if ((error as Error).message.includes('Transform file')) {
|
|
239
|
-
throw error;
|
|
240
|
-
}
|
|
241
|
-
throw new Error(`Failed to load transform file: ${(error as Error).message}`);
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
// =============================================================================
|
|
246
|
-
// Display & Confirmation
|
|
247
|
-
// =============================================================================
|
|
248
|
-
|
|
249
|
-
function displayConfig(config: Config): void {
|
|
250
|
-
console.log('='.repeat(60));
|
|
251
|
-
console.log('🔄 FSCOPY - CONFIGURATION');
|
|
252
|
-
console.log('='.repeat(60));
|
|
253
|
-
console.log('');
|
|
254
|
-
console.log(` 📤 Source project: ${config.sourceProject || '(not set)'}`);
|
|
255
|
-
console.log(` 📥 Destination project: ${config.destProject || '(not set)'}`);
|
|
256
|
-
console.log('');
|
|
257
|
-
console.log(
|
|
258
|
-
` 📋 Collections: ${config.collections.length > 0 ? config.collections.join(', ') : '(none)'}`
|
|
259
|
-
);
|
|
260
|
-
console.log(` 📂 Include subcollections: ${config.includeSubcollections}`);
|
|
261
|
-
console.log(` 🔢 Document limit: ${config.limit === 0 ? 'No limit' : config.limit}`);
|
|
262
|
-
console.log(` 📦 Batch size: ${config.batchSize}`);
|
|
263
|
-
console.log(` 🔄 Retries on error: ${config.retries}`);
|
|
264
|
-
|
|
265
|
-
// New options
|
|
266
|
-
if (config.where.length > 0) {
|
|
267
|
-
const whereStr = config.where.map((w) => `${w.field} ${w.operator} ${w.value}`).join(', ');
|
|
268
|
-
console.log(` 🔍 Where filters: ${whereStr}`);
|
|
269
|
-
}
|
|
270
|
-
if (config.exclude.length > 0) {
|
|
271
|
-
console.log(` 🚫 Exclude patterns: ${config.exclude.join(', ')}`);
|
|
272
|
-
}
|
|
273
|
-
if (config.merge) {
|
|
274
|
-
console.log(` 🔀 Merge mode: enabled (merge instead of overwrite)`);
|
|
275
|
-
}
|
|
276
|
-
if (config.parallel > 1) {
|
|
277
|
-
console.log(` ⚡ Parallel transfers: ${config.parallel} collections`);
|
|
278
|
-
}
|
|
279
|
-
if (config.clear) {
|
|
280
|
-
console.log(` 🗑️ Clear destination: enabled (DESTRUCTIVE)`);
|
|
281
|
-
}
|
|
282
|
-
if (config.deleteMissing) {
|
|
283
|
-
console.log(` 🔄 Delete missing: enabled (sync mode)`);
|
|
284
|
-
}
|
|
285
|
-
if (config.transform) {
|
|
286
|
-
console.log(` 🔧 Transform: ${config.transform}`);
|
|
287
|
-
}
|
|
288
|
-
if (Object.keys(config.renameCollection).length > 0) {
|
|
289
|
-
const renameStr = Object.entries(config.renameCollection)
|
|
290
|
-
.map(([src, dest]) => `${src}→${dest}`)
|
|
291
|
-
.join(', ');
|
|
292
|
-
console.log(` 📝 Rename collections: ${renameStr}`);
|
|
293
|
-
}
|
|
294
|
-
if (config.idPrefix || config.idSuffix) {
|
|
295
|
-
const idMod = [
|
|
296
|
-
config.idPrefix ? `prefix: "${config.idPrefix}"` : null,
|
|
297
|
-
config.idSuffix ? `suffix: "${config.idSuffix}"` : null,
|
|
298
|
-
]
|
|
299
|
-
.filter(Boolean)
|
|
300
|
-
.join(', ');
|
|
301
|
-
console.log(` 🏷️ ID modification: ${idMod}`);
|
|
302
|
-
}
|
|
303
|
-
if (config.rateLimit > 0) {
|
|
304
|
-
console.log(` ⏱️ Rate limit: ${config.rateLimit} docs/s`);
|
|
305
|
-
}
|
|
306
|
-
if (config.skipOversized) {
|
|
307
|
-
console.log(` 📏 Skip oversized: enabled (skip docs > 1MB)`);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
console.log('');
|
|
311
|
-
|
|
312
|
-
if (config.dryRun) {
|
|
313
|
-
console.log(' 🔍 Mode: DRY RUN (no data will be written)');
|
|
314
|
-
} else {
|
|
315
|
-
console.log(' ⚡ Mode: LIVE (data WILL be transferred)');
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
console.log('');
|
|
319
|
-
console.log('='.repeat(60));
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
async function askConfirmation(config: Config): Promise<boolean> {
|
|
323
|
-
const rl = readline.createInterface({
|
|
324
|
-
input: process.stdin,
|
|
325
|
-
output: process.stdout,
|
|
326
|
-
});
|
|
327
|
-
|
|
328
|
-
return new Promise((resolve) => {
|
|
329
|
-
const modeText = config.dryRun ? 'DRY RUN' : '⚠️ LIVE TRANSFER';
|
|
330
|
-
rl.question(`\nProceed with ${modeText}? (y/N): `, (answer) => {
|
|
331
|
-
rl.close();
|
|
332
|
-
resolve(answer.toLowerCase() === 'y' || answer.toLowerCase() === 'yes');
|
|
333
|
-
});
|
|
334
|
-
});
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
// =============================================================================
|
|
338
|
-
// Firebase
|
|
339
|
-
// =============================================================================
|
|
340
|
-
|
|
341
|
-
let sourceApp: admin.app.App | null = null;
|
|
342
|
-
let destApp: admin.app.App | null = null;
|
|
343
|
-
|
|
344
|
-
function initializeFirebase(config: Config): { sourceDb: Firestore; destDb: Firestore } {
|
|
345
|
-
sourceApp = admin.initializeApp(
|
|
346
|
-
{
|
|
347
|
-
credential: admin.credential.applicationDefault(),
|
|
348
|
-
projectId: config.sourceProject!,
|
|
349
|
-
},
|
|
350
|
-
'source'
|
|
351
|
-
);
|
|
352
|
-
|
|
353
|
-
destApp = admin.initializeApp(
|
|
354
|
-
{
|
|
355
|
-
credential: admin.credential.applicationDefault(),
|
|
356
|
-
projectId: config.destProject!,
|
|
357
|
-
},
|
|
358
|
-
'dest'
|
|
359
|
-
);
|
|
360
|
-
|
|
361
|
-
return {
|
|
362
|
-
sourceDb: sourceApp.firestore(),
|
|
363
|
-
destDb: destApp.firestore(),
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
async function checkDatabaseConnectivity(
|
|
368
|
-
sourceDb: Firestore,
|
|
369
|
-
destDb: Firestore,
|
|
370
|
-
config: Config
|
|
371
|
-
): Promise<void> {
|
|
372
|
-
console.log('🔌 Checking database connectivity...');
|
|
373
|
-
|
|
374
|
-
// Check source database
|
|
375
|
-
try {
|
|
376
|
-
await sourceDb.listCollections();
|
|
377
|
-
console.log(` ✓ Source (${config.sourceProject}) - connected`);
|
|
378
|
-
} catch (error) {
|
|
379
|
-
const err = error as Error & { code?: string };
|
|
380
|
-
const errorInfo = formatFirebaseError(err);
|
|
381
|
-
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
382
|
-
throw new Error(`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}`);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
// Check destination database (only if different from source)
|
|
386
|
-
if (config.sourceProject !== config.destProject) {
|
|
387
|
-
try {
|
|
388
|
-
await destDb.listCollections();
|
|
389
|
-
console.log(` ✓ Destination (${config.destProject}) - connected`);
|
|
390
|
-
} catch (error) {
|
|
391
|
-
const err = error as Error & { code?: string };
|
|
392
|
-
const errorInfo = formatFirebaseError(err);
|
|
393
|
-
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
394
|
-
throw new Error(`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}`);
|
|
395
|
-
}
|
|
396
|
-
} else {
|
|
397
|
-
console.log(` ✓ Destination (same as source) - connected`);
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
console.log('');
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
async function cleanupFirebase(): Promise<void> {
|
|
404
|
-
if (sourceApp) await sourceApp.delete();
|
|
405
|
-
if (destApp) await destApp.delete();
|
|
406
|
-
}
|
|
407
236
|
// =============================================================================
|
|
408
237
|
// Main
|
|
409
238
|
// =============================================================================
|
|
410
239
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
// Check credentials before proceeding
|
|
419
|
-
ensureCredentials();
|
|
240
|
+
async function main(): Promise<void> {
|
|
241
|
+
// Handle --init command
|
|
242
|
+
if (argv.init !== undefined) {
|
|
243
|
+
const filename = argv.init || 'fscopy.ini';
|
|
244
|
+
generateConfigFile(filename);
|
|
245
|
+
process.exit(0);
|
|
246
|
+
}
|
|
420
247
|
|
|
421
|
-
//
|
|
422
|
-
|
|
423
|
-
let logger: Logger | null = null;
|
|
424
|
-
let stats: Stats = { collectionsProcessed: 0, documentsTransferred: 0, documentsDeleted: 0, errors: 0 };
|
|
425
|
-
let startTime = Date.now();
|
|
248
|
+
// Check credentials before proceeding
|
|
249
|
+
ensureCredentials();
|
|
426
250
|
|
|
427
|
-
|
|
251
|
+
// Load and merge configuration
|
|
428
252
|
const fileConfig = loadConfigFile(argv.config);
|
|
429
|
-
config = mergeConfig(defaults, fileConfig, argv);
|
|
253
|
+
let config: Config = mergeConfig(defaults, fileConfig, argv);
|
|
430
254
|
|
|
431
255
|
// Run interactive mode if enabled
|
|
432
256
|
if (argv.interactive) {
|
|
@@ -435,6 +259,7 @@ try {
|
|
|
435
259
|
|
|
436
260
|
displayConfig(config);
|
|
437
261
|
|
|
262
|
+
// Validate configuration
|
|
438
263
|
const errors = validateConfig(config);
|
|
439
264
|
if (errors.length > 0) {
|
|
440
265
|
console.log('\n❌ Configuration errors:');
|
|
@@ -454,6 +279,12 @@ try {
|
|
|
454
279
|
}
|
|
455
280
|
}
|
|
456
281
|
|
|
282
|
+
// Exit early if only validating config (for testing)
|
|
283
|
+
if (argv.validateOnly) {
|
|
284
|
+
console.log('\n✓ Configuration is valid');
|
|
285
|
+
process.exit(0);
|
|
286
|
+
}
|
|
287
|
+
|
|
457
288
|
// Skip confirmation in interactive mode (already confirmed by selection)
|
|
458
289
|
if (!argv.yes && !argv.interactive) {
|
|
459
290
|
const confirmed = await askConfirmation(config);
|
|
@@ -463,407 +294,28 @@ try {
|
|
|
463
294
|
}
|
|
464
295
|
}
|
|
465
296
|
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
if (config.resume) {
|
|
473
|
-
const existingState = loadTransferState(config.stateFile);
|
|
474
|
-
if (!existingState) {
|
|
475
|
-
console.error(`\n❌ No state file found at ${config.stateFile}`);
|
|
476
|
-
console.error(' Cannot resume without a saved state. Run without --resume to start fresh.');
|
|
477
|
-
process.exit(1);
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
const stateErrors = validateStateForResume(existingState, config);
|
|
481
|
-
if (stateErrors.length > 0) {
|
|
482
|
-
console.error('\n❌ Cannot resume: state file incompatible with current config:');
|
|
483
|
-
stateErrors.forEach((err) => console.error(` - ${err}`));
|
|
484
|
-
process.exit(1);
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
transferState = existingState;
|
|
488
|
-
const completedCount = Object.values(transferState.completedDocs).reduce((sum, ids) => sum + ids.length, 0);
|
|
489
|
-
console.log(`\n🔄 Resuming transfer from ${config.stateFile}`);
|
|
490
|
-
console.log(` Started: ${transferState.startedAt}`);
|
|
491
|
-
console.log(` Previously completed: ${completedCount} documents`);
|
|
492
|
-
stats = { ...transferState.stats };
|
|
493
|
-
} else if (!config.dryRun) {
|
|
494
|
-
// Create new state for tracking (only in non-dry-run mode)
|
|
495
|
-
transferState = createInitialState(config);
|
|
496
|
-
saveTransferState(config.stateFile, transferState);
|
|
497
|
-
console.log(`\n💾 State will be saved to ${config.stateFile} (use --resume to continue if interrupted)`);
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
// Load transform function if specified
|
|
501
|
-
let transformFn: TransformFunction | null = null;
|
|
502
|
-
if (config.transform) {
|
|
503
|
-
console.log(`\n🔧 Loading transform: ${config.transform}`);
|
|
504
|
-
transformFn = await loadTransformFunction(config.transform);
|
|
505
|
-
console.log(' Transform loaded successfully');
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
console.log('\n');
|
|
509
|
-
startTime = Date.now();
|
|
510
|
-
|
|
511
|
-
const { sourceDb, destDb } = initializeFirebase(config);
|
|
512
|
-
|
|
513
|
-
// Verify database connectivity before proceeding
|
|
514
|
-
await checkDatabaseConnectivity(sourceDb, destDb, config);
|
|
515
|
-
|
|
516
|
-
// Validate transform with sample documents (in dry-run mode)
|
|
517
|
-
if (transformFn && config.dryRun) {
|
|
518
|
-
console.log('🧪 Validating transform with sample documents...');
|
|
519
|
-
let samplesTested = 0;
|
|
520
|
-
let samplesSkipped = 0;
|
|
521
|
-
let samplesErrors = 0;
|
|
522
|
-
|
|
523
|
-
for (const collection of config.collections) {
|
|
524
|
-
const snapshot = await sourceDb.collection(collection).limit(3).get();
|
|
525
|
-
for (const doc of snapshot.docs) {
|
|
526
|
-
try {
|
|
527
|
-
const result = transformFn(doc.data() as Record<string, unknown>, {
|
|
528
|
-
id: doc.id,
|
|
529
|
-
path: `${collection}/${doc.id}`,
|
|
530
|
-
});
|
|
531
|
-
if (result === null) {
|
|
532
|
-
samplesSkipped++;
|
|
533
|
-
} else {
|
|
534
|
-
samplesTested++;
|
|
535
|
-
}
|
|
536
|
-
} catch (error) {
|
|
537
|
-
samplesErrors++;
|
|
538
|
-
const err = error as Error;
|
|
539
|
-
console.error(` ⚠️ Transform error on ${collection}/${doc.id}: ${err.message}`);
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
if (samplesErrors > 0) {
|
|
545
|
-
console.log(` ❌ ${samplesErrors} sample(s) failed - review your transform function`);
|
|
546
|
-
} else if (samplesTested > 0 || samplesSkipped > 0) {
|
|
547
|
-
console.log(` ✓ Tested ${samplesTested} sample(s), ${samplesSkipped} would be skipped`);
|
|
548
|
-
}
|
|
549
|
-
console.log('');
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
if (!config.resume) {
|
|
553
|
-
stats = {
|
|
554
|
-
collectionsProcessed: 0,
|
|
555
|
-
documentsTransferred: 0,
|
|
556
|
-
documentsDeleted: 0,
|
|
557
|
-
errors: 0,
|
|
558
|
-
};
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
// Count total documents for progress bar
|
|
562
|
-
let totalDocs = 0;
|
|
563
|
-
let progressBar: cliProgress.SingleBar | null = null;
|
|
564
|
-
|
|
565
|
-
if (!argv.quiet) {
|
|
566
|
-
console.log('📊 Counting documents...');
|
|
567
|
-
let lastSubcollectionLog = Date.now();
|
|
568
|
-
let subcollectionCount = 0;
|
|
569
|
-
|
|
570
|
-
const countProgress: CountProgress = {
|
|
571
|
-
onCollection: (path, count) => {
|
|
572
|
-
console.log(` ${path}: ${count} documents`);
|
|
573
|
-
},
|
|
574
|
-
onSubcollection: (_path) => {
|
|
575
|
-
subcollectionCount++;
|
|
576
|
-
// Show progress every 2 seconds to avoid flooding the console
|
|
577
|
-
const now = Date.now();
|
|
578
|
-
if (now - lastSubcollectionLog > 2000) {
|
|
579
|
-
process.stdout.write(`\r Scanning subcollections... (${subcollectionCount} found)`);
|
|
580
|
-
lastSubcollectionLog = now;
|
|
581
|
-
}
|
|
582
|
-
},
|
|
583
|
-
};
|
|
584
|
-
|
|
585
|
-
for (const collection of config.collections) {
|
|
586
|
-
totalDocs += await countDocuments(sourceDb, collection, config, 0, countProgress);
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
// Clear the subcollection line if any were found
|
|
590
|
-
if (subcollectionCount > 0) {
|
|
591
|
-
process.stdout.write('\r' + ' '.repeat(60) + '\r');
|
|
592
|
-
console.log(` Subcollections scanned: ${subcollectionCount}`);
|
|
593
|
-
}
|
|
594
|
-
console.log(` Total: ${totalDocs} documents to transfer\n`);
|
|
595
|
-
|
|
596
|
-
if (totalDocs > 0) {
|
|
597
|
-
progressBar = new cliProgress.SingleBar({
|
|
598
|
-
format: '📦 Progress |{bar}| {percentage}% | {value}/{total} docs | {speed} docs/s | ETA: {eta}s',
|
|
599
|
-
barCompleteChar: '█',
|
|
600
|
-
barIncompleteChar: '░',
|
|
601
|
-
hideCursor: true,
|
|
602
|
-
});
|
|
603
|
-
progressBar.start(totalDocs, 0, { speed: '0' });
|
|
604
|
-
|
|
605
|
-
// Track speed using transfer stats
|
|
606
|
-
let lastDocsTransferred = 0;
|
|
607
|
-
let lastTime = Date.now();
|
|
608
|
-
|
|
609
|
-
const speedInterval = setInterval(() => {
|
|
610
|
-
if (progressBar) {
|
|
611
|
-
const now = Date.now();
|
|
612
|
-
const timeDiff = (now - lastTime) / 1000;
|
|
613
|
-
const currentDocs = stats.documentsTransferred;
|
|
614
|
-
|
|
615
|
-
if (timeDiff > 0) {
|
|
616
|
-
const docsDiff = currentDocs - lastDocsTransferred;
|
|
617
|
-
const speed = Math.round(docsDiff / timeDiff);
|
|
618
|
-
lastDocsTransferred = currentDocs;
|
|
619
|
-
lastTime = now;
|
|
620
|
-
progressBar.update({ speed: String(speed) });
|
|
621
|
-
}
|
|
622
|
-
}
|
|
623
|
-
}, 500);
|
|
624
|
-
|
|
625
|
-
// Store interval for cleanup
|
|
626
|
-
(progressBar as unknown as { _speedInterval: NodeJS.Timeout })._speedInterval = speedInterval;
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
|
|
630
|
-
// Clear destination collections if enabled
|
|
631
|
-
if (config.clear) {
|
|
632
|
-
console.log('🗑️ Clearing destination collections...');
|
|
633
|
-
for (const collection of config.collections) {
|
|
634
|
-
const destCollection = getDestCollectionPath(collection, config.renameCollection);
|
|
635
|
-
const deleted = await clearCollection(
|
|
636
|
-
destDb,
|
|
637
|
-
destCollection,
|
|
638
|
-
config,
|
|
639
|
-
logger,
|
|
640
|
-
config.includeSubcollections
|
|
641
|
-
);
|
|
642
|
-
stats.documentsDeleted += deleted;
|
|
643
|
-
}
|
|
644
|
-
console.log(` Deleted ${stats.documentsDeleted} documents\n`);
|
|
645
|
-
}
|
|
646
|
-
|
|
647
|
-
// Create rate limiter if enabled
|
|
648
|
-
const rateLimiter = config.rateLimit > 0 ? new RateLimiter(config.rateLimit) : null;
|
|
649
|
-
if (rateLimiter) {
|
|
650
|
-
console.log(`⏱️ Rate limiting enabled: ${config.rateLimit} docs/s\n`);
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
const ctx: TransferContext = { sourceDb, destDb, config, stats, logger, progressBar, transformFn, state: transferState, rateLimiter };
|
|
654
|
-
|
|
655
|
-
// Transfer collections (with optional parallelism)
|
|
656
|
-
if (config.parallel > 1) {
|
|
657
|
-
const { errors } = await processInParallel(config.collections, config.parallel, (collection) =>
|
|
658
|
-
transferCollection(ctx, collection)
|
|
659
|
-
);
|
|
660
|
-
if (errors.length > 0) {
|
|
661
|
-
for (const err of errors) {
|
|
662
|
-
logger.error('Parallel transfer error', { error: err.message });
|
|
663
|
-
stats.errors++;
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
} else {
|
|
667
|
-
for (const collection of config.collections) {
|
|
668
|
-
try {
|
|
669
|
-
await transferCollection(ctx, collection);
|
|
670
|
-
} catch (error) {
|
|
671
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
672
|
-
logger.error(`Transfer failed for ${collection}`, { error: err.message });
|
|
673
|
-
stats.errors++;
|
|
674
|
-
}
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
if (progressBar) {
|
|
679
|
-
// Clear speed update interval
|
|
680
|
-
const interval = (progressBar as unknown as { _speedInterval?: NodeJS.Timeout })._speedInterval;
|
|
681
|
-
if (interval) {
|
|
682
|
-
clearInterval(interval);
|
|
683
|
-
}
|
|
684
|
-
progressBar.stop();
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
// Delete orphan documents if enabled (sync mode)
|
|
688
|
-
if (config.deleteMissing) {
|
|
689
|
-
console.log('\n🔄 Deleting orphan documents (sync mode)...');
|
|
690
|
-
for (const collection of config.collections) {
|
|
691
|
-
const deleted = await deleteOrphanDocuments(
|
|
692
|
-
sourceDb,
|
|
693
|
-
destDb,
|
|
694
|
-
collection,
|
|
695
|
-
config,
|
|
696
|
-
logger
|
|
697
|
-
);
|
|
698
|
-
stats.documentsDeleted += deleted;
|
|
699
|
-
}
|
|
700
|
-
if (stats.documentsDeleted > 0) {
|
|
701
|
-
console.log(` Deleted ${stats.documentsDeleted} orphan documents`);
|
|
702
|
-
} else {
|
|
703
|
-
console.log(' No orphan documents found');
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
|
|
708
|
-
|
|
709
|
-
logger.success('Transfer completed', {
|
|
710
|
-
stats: stats as unknown as Record<string, unknown>,
|
|
711
|
-
duration,
|
|
297
|
+
// Initialize output
|
|
298
|
+
const output = new Output({
|
|
299
|
+
quiet: argv.quiet,
|
|
300
|
+
json: argv.json,
|
|
301
|
+
logFile: argv.log,
|
|
302
|
+
maxLogSize: parseSize(argv.maxLogSize),
|
|
712
303
|
});
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
// Verify transfer if enabled (and not dry-run)
|
|
716
|
-
let verifyResult: Record<string, { source: number; dest: number; match: boolean }> | null = null;
|
|
717
|
-
if (config.verify && !config.dryRun) {
|
|
718
|
-
if (!config.json) {
|
|
719
|
-
console.log('\n🔍 Verifying transfer...');
|
|
720
|
-
}
|
|
721
|
-
verifyResult = {};
|
|
722
|
-
let verifyPassed = true;
|
|
723
|
-
|
|
724
|
-
for (const collection of config.collections) {
|
|
725
|
-
const destCollection = getDestCollectionPath(collection, config.renameCollection);
|
|
726
|
-
|
|
727
|
-
// Count source documents
|
|
728
|
-
const sourceCount = await sourceDb.collection(collection).count().get();
|
|
729
|
-
const sourceTotal = sourceCount.data().count;
|
|
730
|
-
|
|
731
|
-
// Count destination documents
|
|
732
|
-
const destCount = await destDb.collection(destCollection).count().get();
|
|
733
|
-
const destTotal = destCount.data().count;
|
|
734
|
-
|
|
735
|
-
const match = sourceTotal === destTotal;
|
|
736
|
-
verifyResult[collection] = { source: sourceTotal, dest: destTotal, match };
|
|
737
|
-
|
|
738
|
-
if (!config.json) {
|
|
739
|
-
if (match) {
|
|
740
|
-
console.log(` ✓ ${collection}: ${sourceTotal} docs (matched)`);
|
|
741
|
-
} else {
|
|
742
|
-
console.log(` ⚠️ ${collection}: source=${sourceTotal}, dest=${destTotal} (mismatch)`);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
if (!match) verifyPassed = false;
|
|
746
|
-
}
|
|
304
|
+
output.init();
|
|
305
|
+
output.logInfo('Transfer started', { config: config as unknown as Record<string, unknown> });
|
|
747
306
|
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
console.log(' ✓ Verification passed');
|
|
751
|
-
} else {
|
|
752
|
-
console.log(' ⚠️ Verification found mismatches');
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
// Delete state file on successful completion (before JSON output)
|
|
758
|
-
if (!config.dryRun) {
|
|
759
|
-
deleteTransferState(config.stateFile);
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
// JSON output mode
|
|
763
|
-
if (config.json) {
|
|
764
|
-
const jsonOutput = {
|
|
765
|
-
success: true,
|
|
766
|
-
dryRun: config.dryRun,
|
|
767
|
-
source: config.sourceProject,
|
|
768
|
-
destination: config.destProject,
|
|
769
|
-
collections: config.collections,
|
|
770
|
-
stats: {
|
|
771
|
-
collectionsProcessed: stats.collectionsProcessed,
|
|
772
|
-
documentsTransferred: stats.documentsTransferred,
|
|
773
|
-
documentsDeleted: stats.documentsDeleted,
|
|
774
|
-
errors: stats.errors,
|
|
775
|
-
},
|
|
776
|
-
duration: Number.parseFloat(duration),
|
|
777
|
-
verify: verifyResult,
|
|
778
|
-
};
|
|
779
|
-
console.log(JSON.stringify(jsonOutput, null, 2));
|
|
780
|
-
} else {
|
|
781
|
-
console.log('\n' + '='.repeat(60));
|
|
782
|
-
console.log('📊 TRANSFER SUMMARY');
|
|
783
|
-
console.log('='.repeat(60));
|
|
784
|
-
console.log(`Collections processed: ${stats.collectionsProcessed}`);
|
|
785
|
-
if (stats.documentsDeleted > 0) {
|
|
786
|
-
console.log(`Documents deleted: ${stats.documentsDeleted}`);
|
|
787
|
-
}
|
|
788
|
-
console.log(`Documents transferred: ${stats.documentsTransferred}`);
|
|
789
|
-
console.log(`Errors: ${stats.errors}`);
|
|
790
|
-
console.log(`Duration: ${duration}s`);
|
|
791
|
-
|
|
792
|
-
if (argv.log) {
|
|
793
|
-
console.log(`Log file: ${argv.log}`);
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
if (config.dryRun) {
|
|
797
|
-
console.log('\n⚠ DRY RUN: No data was actually written');
|
|
798
|
-
console.log(' Run with --dry-run=false to perform the transfer');
|
|
799
|
-
} else {
|
|
800
|
-
console.log('\n✓ Transfer completed successfully');
|
|
801
|
-
}
|
|
802
|
-
console.log('='.repeat(60) + '\n');
|
|
803
|
-
}
|
|
307
|
+
// Run transfer
|
|
308
|
+
const result = await runTransfer(config, argv, output);
|
|
804
309
|
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
await sendWebhook(
|
|
808
|
-
config.webhook,
|
|
809
|
-
{
|
|
810
|
-
source: config.sourceProject!,
|
|
811
|
-
destination: config.destProject!,
|
|
812
|
-
collections: config.collections,
|
|
813
|
-
stats,
|
|
814
|
-
duration: Number.parseFloat(duration),
|
|
815
|
-
dryRun: config.dryRun,
|
|
816
|
-
success: true,
|
|
817
|
-
},
|
|
818
|
-
logger
|
|
819
|
-
);
|
|
310
|
+
if (!result.success) {
|
|
311
|
+
process.exit(1);
|
|
820
312
|
}
|
|
313
|
+
}
|
|
821
314
|
|
|
822
|
-
|
|
315
|
+
// Run main
|
|
316
|
+
try {
|
|
317
|
+
await main();
|
|
823
318
|
} catch (error) {
|
|
824
|
-
|
|
825
|
-
const duration = Number.parseFloat(((Date.now() - startTime) / 1000).toFixed(2));
|
|
826
|
-
|
|
827
|
-
// JSON output mode for errors
|
|
828
|
-
if (config.json) {
|
|
829
|
-
const jsonOutput = {
|
|
830
|
-
success: false,
|
|
831
|
-
error: errorMessage,
|
|
832
|
-
dryRun: config.dryRun,
|
|
833
|
-
source: config.sourceProject,
|
|
834
|
-
destination: config.destProject,
|
|
835
|
-
collections: config.collections,
|
|
836
|
-
stats: {
|
|
837
|
-
collectionsProcessed: stats.collectionsProcessed,
|
|
838
|
-
documentsTransferred: stats.documentsTransferred,
|
|
839
|
-
documentsDeleted: stats.documentsDeleted,
|
|
840
|
-
errors: stats.errors,
|
|
841
|
-
},
|
|
842
|
-
duration,
|
|
843
|
-
};
|
|
844
|
-
console.log(JSON.stringify(jsonOutput, null, 2));
|
|
845
|
-
} else {
|
|
846
|
-
console.error('\n❌ Error during transfer:', errorMessage);
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
// Send webhook notification on error if configured
|
|
850
|
-
if (config.webhook && logger) {
|
|
851
|
-
await sendWebhook(
|
|
852
|
-
config.webhook,
|
|
853
|
-
{
|
|
854
|
-
source: config.sourceProject ?? 'unknown',
|
|
855
|
-
destination: config.destProject ?? 'unknown',
|
|
856
|
-
collections: config.collections,
|
|
857
|
-
stats,
|
|
858
|
-
duration,
|
|
859
|
-
dryRun: config.dryRun,
|
|
860
|
-
success: false,
|
|
861
|
-
error: errorMessage,
|
|
862
|
-
},
|
|
863
|
-
logger
|
|
864
|
-
);
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
await cleanupFirebase();
|
|
319
|
+
console.error('\n❌ Unexpected error:', (error as Error).message);
|
|
868
320
|
process.exit(1);
|
|
869
321
|
}
|