@fazetitans/fscopy 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -33
- package/package.json +3 -3
- package/src/cli.ts +82 -620
- package/src/config/defaults.ts +4 -0
- package/src/config/parser.ts +4 -0
- package/src/config/validator.ts +52 -0
- package/src/firebase/index.ts +82 -0
- package/src/interactive.ts +59 -56
- package/src/orchestrator.ts +407 -0
- package/src/output/display.ts +221 -0
- package/src/state/index.ts +188 -1
- package/src/transfer/clear.ts +162 -104
- package/src/transfer/count.ts +83 -44
- package/src/transfer/transfer.ts +487 -156
- package/src/transform/loader.ts +31 -0
- package/src/types.ts +18 -0
- package/src/utils/credentials.ts +9 -4
- package/src/utils/doc-size.ts +41 -70
- package/src/utils/errors.ts +1 -1
- package/src/utils/index.ts +2 -1
- package/src/utils/integrity.ts +122 -0
- package/src/utils/logger.ts +59 -3
- package/src/utils/output.ts +265 -0
- package/src/utils/patterns.ts +3 -2
- package/src/utils/progress.ts +102 -0
- package/src/utils/rate-limiter.ts +4 -2
- package/src/webhook/index.ts +24 -6
package/src/cli.ts
CHANGED
|
@@ -3,29 +3,20 @@
|
|
|
3
3
|
// Suppress GCE metadata lookup warning (we're not running on Google Cloud)
|
|
4
4
|
process.env.METADATA_SERVER_DETECTION = 'none';
|
|
5
5
|
|
|
6
|
-
import admin from 'firebase-admin';
|
|
7
|
-
import type { Firestore } from 'firebase-admin/firestore';
|
|
8
6
|
import yargs from 'yargs';
|
|
9
7
|
import { hideBin } from 'yargs/helpers';
|
|
10
|
-
import cliProgress from 'cli-progress';
|
|
11
|
-
import fs from 'node:fs';
|
|
12
|
-
import path from 'node:path';
|
|
13
|
-
import readline from 'node:readline';
|
|
14
8
|
|
|
15
|
-
|
|
16
|
-
import
|
|
17
|
-
import { Logger } from './utils/logger.js';
|
|
9
|
+
import type { Config, CliArgs } from './types.js';
|
|
10
|
+
import { Output, parseSize } from './utils/output.js';
|
|
18
11
|
import { ensureCredentials } from './utils/credentials.js';
|
|
19
|
-
import { formatFirebaseError } from './utils/errors.js';
|
|
20
|
-
import { RateLimiter } from './utils/rate-limiter.js';
|
|
21
12
|
import { loadConfigFile, mergeConfig } from './config/parser.js';
|
|
22
13
|
import { validateConfig } from './config/validator.js';
|
|
23
14
|
import { defaults } from './config/defaults.js';
|
|
24
15
|
import { generateConfigFile } from './config/generator.js';
|
|
25
|
-
import {
|
|
26
|
-
import { sendWebhook } from './webhook/index.js';
|
|
27
|
-
import { countDocuments, transferCollection, clearCollection, deleteOrphanDocuments, processInParallel, getDestCollectionPath, type TransferContext, type CountProgress } from './transfer/index.js';
|
|
16
|
+
import { validateWebhookUrl } from './webhook/index.js';
|
|
28
17
|
import { runInteractiveMode } from './interactive.js';
|
|
18
|
+
import { displayConfig, askConfirmation } from './output/display.js';
|
|
19
|
+
import { runTransfer } from './orchestrator.js';
|
|
29
20
|
|
|
30
21
|
// =============================================================================
|
|
31
22
|
// CLI Arguments
|
|
@@ -88,6 +79,11 @@ const argv = yargs(hideBin(process.argv))
|
|
|
88
79
|
type: 'string',
|
|
89
80
|
description: 'Path to log file for transfer details',
|
|
90
81
|
})
|
|
82
|
+
.option('max-log-size', {
|
|
83
|
+
type: 'string',
|
|
84
|
+
description: 'Max log file size before rotation (e.g., "10MB", "1GB"). 0 = no rotation.',
|
|
85
|
+
default: '0',
|
|
86
|
+
})
|
|
91
87
|
.option('retries', {
|
|
92
88
|
type: 'number',
|
|
93
89
|
description: 'Number of retries on error (default: 3)',
|
|
@@ -189,6 +185,32 @@ const argv = yargs(hideBin(process.argv))
|
|
|
189
185
|
description: 'Output results in JSON format (for CI/CD)',
|
|
190
186
|
default: false,
|
|
191
187
|
})
|
|
188
|
+
.option('transform-samples', {
|
|
189
|
+
type: 'number',
|
|
190
|
+
description: 'Number of documents to test per collection during transform validation (0 = skip, -1 = all)',
|
|
191
|
+
default: 3,
|
|
192
|
+
})
|
|
193
|
+
.option('detect-conflicts', {
|
|
194
|
+
type: 'boolean',
|
|
195
|
+
description: 'Detect if destination docs were modified during transfer',
|
|
196
|
+
default: false,
|
|
197
|
+
})
|
|
198
|
+
.option('max-depth', {
|
|
199
|
+
type: 'number',
|
|
200
|
+
description: 'Max subcollection depth (0 = unlimited)',
|
|
201
|
+
default: 0,
|
|
202
|
+
})
|
|
203
|
+
.option('verify-integrity', {
|
|
204
|
+
type: 'boolean',
|
|
205
|
+
description: 'Verify document integrity with hash after transfer',
|
|
206
|
+
default: false,
|
|
207
|
+
})
|
|
208
|
+
.option('validate-only', {
|
|
209
|
+
type: 'boolean',
|
|
210
|
+
description: 'Only validate config and display it (no transfer)',
|
|
211
|
+
default: false,
|
|
212
|
+
hidden: true,
|
|
213
|
+
})
|
|
192
214
|
.example('$0 --init config.ini', 'Generate INI config template (default)')
|
|
193
215
|
.example('$0 --init config.json', 'Generate JSON config template')
|
|
194
216
|
.example('$0 -f config.ini', 'Run transfer with config file')
|
|
@@ -209,224 +231,24 @@ const argv = yargs(hideBin(process.argv))
|
|
|
209
231
|
.help()
|
|
210
232
|
.parseSync() as CliArgs;
|
|
211
233
|
|
|
212
|
-
|
|
213
|
-
// =============================================================================
|
|
214
|
-
// Transform Loading
|
|
215
|
-
// =============================================================================
|
|
216
|
-
|
|
217
|
-
async function loadTransformFunction(transformPath: string): Promise<TransformFunction> {
|
|
218
|
-
const absolutePath = path.resolve(transformPath);
|
|
219
|
-
|
|
220
|
-
if (!fs.existsSync(absolutePath)) {
|
|
221
|
-
throw new Error(`Transform file not found: ${absolutePath}`);
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
try {
|
|
225
|
-
const module = await import(absolutePath);
|
|
226
|
-
|
|
227
|
-
// Look for 'transform' export (default or named)
|
|
228
|
-
const transformFn = module.default?.transform ?? module.transform ?? module.default;
|
|
229
|
-
|
|
230
|
-
if (typeof transformFn !== 'function') {
|
|
231
|
-
throw new Error(
|
|
232
|
-
`Transform file must export a 'transform' function. Got: ${typeof transformFn}`
|
|
233
|
-
);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
return transformFn as TransformFunction;
|
|
237
|
-
} catch (error) {
|
|
238
|
-
if ((error as Error).message.includes('Transform file')) {
|
|
239
|
-
throw error;
|
|
240
|
-
}
|
|
241
|
-
throw new Error(`Failed to load transform file: ${(error as Error).message}`);
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
// =============================================================================
|
|
246
|
-
// Display & Confirmation
|
|
247
|
-
// =============================================================================
|
|
248
|
-
|
|
249
|
-
function displayConfig(config: Config): void {
|
|
250
|
-
console.log('='.repeat(60));
|
|
251
|
-
console.log('🔄 FSCOPY - CONFIGURATION');
|
|
252
|
-
console.log('='.repeat(60));
|
|
253
|
-
console.log('');
|
|
254
|
-
console.log(` 📤 Source project: ${config.sourceProject || '(not set)'}`);
|
|
255
|
-
console.log(` 📥 Destination project: ${config.destProject || '(not set)'}`);
|
|
256
|
-
console.log('');
|
|
257
|
-
console.log(
|
|
258
|
-
` 📋 Collections: ${config.collections.length > 0 ? config.collections.join(', ') : '(none)'}`
|
|
259
|
-
);
|
|
260
|
-
console.log(` 📂 Include subcollections: ${config.includeSubcollections}`);
|
|
261
|
-
console.log(` 🔢 Document limit: ${config.limit === 0 ? 'No limit' : config.limit}`);
|
|
262
|
-
console.log(` 📦 Batch size: ${config.batchSize}`);
|
|
263
|
-
console.log(` 🔄 Retries on error: ${config.retries}`);
|
|
264
|
-
|
|
265
|
-
// New options
|
|
266
|
-
if (config.where.length > 0) {
|
|
267
|
-
const whereStr = config.where.map((w) => `${w.field} ${w.operator} ${w.value}`).join(', ');
|
|
268
|
-
console.log(` 🔍 Where filters: ${whereStr}`);
|
|
269
|
-
}
|
|
270
|
-
if (config.exclude.length > 0) {
|
|
271
|
-
console.log(` 🚫 Exclude patterns: ${config.exclude.join(', ')}`);
|
|
272
|
-
}
|
|
273
|
-
if (config.merge) {
|
|
274
|
-
console.log(` 🔀 Merge mode: enabled (merge instead of overwrite)`);
|
|
275
|
-
}
|
|
276
|
-
if (config.parallel > 1) {
|
|
277
|
-
console.log(` ⚡ Parallel transfers: ${config.parallel} collections`);
|
|
278
|
-
}
|
|
279
|
-
if (config.clear) {
|
|
280
|
-
console.log(` 🗑️ Clear destination: enabled (DESTRUCTIVE)`);
|
|
281
|
-
}
|
|
282
|
-
if (config.deleteMissing) {
|
|
283
|
-
console.log(` 🔄 Delete missing: enabled (sync mode)`);
|
|
284
|
-
}
|
|
285
|
-
if (config.transform) {
|
|
286
|
-
console.log(` 🔧 Transform: ${config.transform}`);
|
|
287
|
-
}
|
|
288
|
-
if (Object.keys(config.renameCollection).length > 0) {
|
|
289
|
-
const renameStr = Object.entries(config.renameCollection)
|
|
290
|
-
.map(([src, dest]) => `${src}→${dest}`)
|
|
291
|
-
.join(', ');
|
|
292
|
-
console.log(` 📝 Rename collections: ${renameStr}`);
|
|
293
|
-
}
|
|
294
|
-
if (config.idPrefix || config.idSuffix) {
|
|
295
|
-
const idMod = [
|
|
296
|
-
config.idPrefix ? `prefix: "${config.idPrefix}"` : null,
|
|
297
|
-
config.idSuffix ? `suffix: "${config.idSuffix}"` : null,
|
|
298
|
-
]
|
|
299
|
-
.filter(Boolean)
|
|
300
|
-
.join(', ');
|
|
301
|
-
console.log(` 🏷️ ID modification: ${idMod}`);
|
|
302
|
-
}
|
|
303
|
-
if (config.rateLimit > 0) {
|
|
304
|
-
console.log(` ⏱️ Rate limit: ${config.rateLimit} docs/s`);
|
|
305
|
-
}
|
|
306
|
-
if (config.skipOversized) {
|
|
307
|
-
console.log(` 📏 Skip oversized: enabled (skip docs > 1MB)`);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
console.log('');
|
|
311
|
-
|
|
312
|
-
if (config.dryRun) {
|
|
313
|
-
console.log(' 🔍 Mode: DRY RUN (no data will be written)');
|
|
314
|
-
} else {
|
|
315
|
-
console.log(' ⚡ Mode: LIVE (data WILL be transferred)');
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
console.log('');
|
|
319
|
-
console.log('='.repeat(60));
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
async function askConfirmation(config: Config): Promise<boolean> {
|
|
323
|
-
const rl = readline.createInterface({
|
|
324
|
-
input: process.stdin,
|
|
325
|
-
output: process.stdout,
|
|
326
|
-
});
|
|
327
|
-
|
|
328
|
-
return new Promise((resolve) => {
|
|
329
|
-
const modeText = config.dryRun ? 'DRY RUN' : '⚠️ LIVE TRANSFER';
|
|
330
|
-
rl.question(`\nProceed with ${modeText}? (y/N): `, (answer) => {
|
|
331
|
-
rl.close();
|
|
332
|
-
resolve(answer.toLowerCase() === 'y' || answer.toLowerCase() === 'yes');
|
|
333
|
-
});
|
|
334
|
-
});
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
// =============================================================================
|
|
338
|
-
// Firebase
|
|
339
|
-
// =============================================================================
|
|
340
|
-
|
|
341
|
-
let sourceApp: admin.app.App | null = null;
|
|
342
|
-
let destApp: admin.app.App | null = null;
|
|
343
|
-
|
|
344
|
-
function initializeFirebase(config: Config): { sourceDb: Firestore; destDb: Firestore } {
|
|
345
|
-
sourceApp = admin.initializeApp(
|
|
346
|
-
{
|
|
347
|
-
credential: admin.credential.applicationDefault(),
|
|
348
|
-
projectId: config.sourceProject!,
|
|
349
|
-
},
|
|
350
|
-
'source'
|
|
351
|
-
);
|
|
352
|
-
|
|
353
|
-
destApp = admin.initializeApp(
|
|
354
|
-
{
|
|
355
|
-
credential: admin.credential.applicationDefault(),
|
|
356
|
-
projectId: config.destProject!,
|
|
357
|
-
},
|
|
358
|
-
'dest'
|
|
359
|
-
);
|
|
360
|
-
|
|
361
|
-
return {
|
|
362
|
-
sourceDb: sourceApp.firestore(),
|
|
363
|
-
destDb: destApp.firestore(),
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
async function checkDatabaseConnectivity(
|
|
368
|
-
sourceDb: Firestore,
|
|
369
|
-
destDb: Firestore,
|
|
370
|
-
config: Config
|
|
371
|
-
): Promise<void> {
|
|
372
|
-
console.log('🔌 Checking database connectivity...');
|
|
373
|
-
|
|
374
|
-
// Check source database
|
|
375
|
-
try {
|
|
376
|
-
await sourceDb.listCollections();
|
|
377
|
-
console.log(` ✓ Source (${config.sourceProject}) - connected`);
|
|
378
|
-
} catch (error) {
|
|
379
|
-
const err = error as Error & { code?: string };
|
|
380
|
-
const errorInfo = formatFirebaseError(err);
|
|
381
|
-
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
382
|
-
throw new Error(`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}`);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
// Check destination database (only if different from source)
|
|
386
|
-
if (config.sourceProject !== config.destProject) {
|
|
387
|
-
try {
|
|
388
|
-
await destDb.listCollections();
|
|
389
|
-
console.log(` ✓ Destination (${config.destProject}) - connected`);
|
|
390
|
-
} catch (error) {
|
|
391
|
-
const err = error as Error & { code?: string };
|
|
392
|
-
const errorInfo = formatFirebaseError(err);
|
|
393
|
-
const hint = errorInfo.suggestion ? `\n Hint: ${errorInfo.suggestion}` : '';
|
|
394
|
-
throw new Error(`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}`);
|
|
395
|
-
}
|
|
396
|
-
} else {
|
|
397
|
-
console.log(` ✓ Destination (same as source) - connected`);
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
console.log('');
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
async function cleanupFirebase(): Promise<void> {
|
|
404
|
-
if (sourceApp) await sourceApp.delete();
|
|
405
|
-
if (destApp) await destApp.delete();
|
|
406
|
-
}
|
|
407
234
|
// =============================================================================
|
|
408
235
|
// Main
|
|
409
236
|
// =============================================================================
|
|
410
237
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
// Check credentials before proceeding
|
|
419
|
-
ensureCredentials();
|
|
238
|
+
async function main(): Promise<void> {
|
|
239
|
+
// Handle --init command
|
|
240
|
+
if (argv.init !== undefined) {
|
|
241
|
+
const filename = argv.init || 'fscopy.ini';
|
|
242
|
+
generateConfigFile(filename);
|
|
243
|
+
process.exit(0);
|
|
244
|
+
}
|
|
420
245
|
|
|
421
|
-
//
|
|
422
|
-
|
|
423
|
-
let logger: Logger | null = null;
|
|
424
|
-
let stats: Stats = { collectionsProcessed: 0, documentsTransferred: 0, documentsDeleted: 0, errors: 0 };
|
|
425
|
-
let startTime = Date.now();
|
|
246
|
+
// Check credentials before proceeding
|
|
247
|
+
ensureCredentials();
|
|
426
248
|
|
|
427
|
-
|
|
249
|
+
// Load and merge configuration
|
|
428
250
|
const fileConfig = loadConfigFile(argv.config);
|
|
429
|
-
config = mergeConfig(defaults, fileConfig, argv);
|
|
251
|
+
let config: Config = mergeConfig(defaults, fileConfig, argv);
|
|
430
252
|
|
|
431
253
|
// Run interactive mode if enabled
|
|
432
254
|
if (argv.interactive) {
|
|
@@ -435,6 +257,7 @@ try {
|
|
|
435
257
|
|
|
436
258
|
displayConfig(config);
|
|
437
259
|
|
|
260
|
+
// Validate configuration
|
|
438
261
|
const errors = validateConfig(config);
|
|
439
262
|
if (errors.length > 0) {
|
|
440
263
|
console.log('\n❌ Configuration errors:');
|
|
@@ -442,416 +265,55 @@ try {
|
|
|
442
265
|
process.exit(1);
|
|
443
266
|
}
|
|
444
267
|
|
|
445
|
-
//
|
|
446
|
-
if (
|
|
447
|
-
const
|
|
448
|
-
if (!
|
|
449
|
-
console.log(
|
|
450
|
-
process.exit(0);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
logger = new Logger(argv.log);
|
|
455
|
-
logger.init();
|
|
456
|
-
logger.info('Transfer started', { config: config as unknown as Record<string, unknown> });
|
|
457
|
-
|
|
458
|
-
// Handle resume mode
|
|
459
|
-
let transferState: TransferState | null = null;
|
|
460
|
-
if (config.resume) {
|
|
461
|
-
const existingState = loadTransferState(config.stateFile);
|
|
462
|
-
if (!existingState) {
|
|
463
|
-
console.error(`\n❌ No state file found at ${config.stateFile}`);
|
|
464
|
-
console.error(' Cannot resume without a saved state. Run without --resume to start fresh.');
|
|
465
|
-
process.exit(1);
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
const stateErrors = validateStateForResume(existingState, config);
|
|
469
|
-
if (stateErrors.length > 0) {
|
|
470
|
-
console.error('\n❌ Cannot resume: state file incompatible with current config:');
|
|
471
|
-
stateErrors.forEach((err) => console.error(` - ${err}`));
|
|
268
|
+
// Validate webhook URL if configured
|
|
269
|
+
if (config.webhook) {
|
|
270
|
+
const webhookValidation = validateWebhookUrl(config.webhook);
|
|
271
|
+
if (!webhookValidation.valid) {
|
|
272
|
+
console.log(`\n❌ ${webhookValidation.warning}`);
|
|
472
273
|
process.exit(1);
|
|
473
274
|
}
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
const completedCount = Object.values(transferState.completedDocs).reduce((sum, ids) => sum + ids.length, 0);
|
|
477
|
-
console.log(`\n🔄 Resuming transfer from ${config.stateFile}`);
|
|
478
|
-
console.log(` Started: ${transferState.startedAt}`);
|
|
479
|
-
console.log(` Previously completed: ${completedCount} documents`);
|
|
480
|
-
stats = { ...transferState.stats };
|
|
481
|
-
} else if (!config.dryRun) {
|
|
482
|
-
// Create new state for tracking (only in non-dry-run mode)
|
|
483
|
-
transferState = createInitialState(config);
|
|
484
|
-
saveTransferState(config.stateFile, transferState);
|
|
485
|
-
console.log(`\n💾 State will be saved to ${config.stateFile} (use --resume to continue if interrupted)`);
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
// Load transform function if specified
|
|
489
|
-
let transformFn: TransformFunction | null = null;
|
|
490
|
-
if (config.transform) {
|
|
491
|
-
console.log(`\n🔧 Loading transform: ${config.transform}`);
|
|
492
|
-
transformFn = await loadTransformFunction(config.transform);
|
|
493
|
-
console.log(' Transform loaded successfully');
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
console.log('\n');
|
|
497
|
-
startTime = Date.now();
|
|
498
|
-
|
|
499
|
-
const { sourceDb, destDb } = initializeFirebase(config);
|
|
500
|
-
|
|
501
|
-
// Verify database connectivity before proceeding
|
|
502
|
-
await checkDatabaseConnectivity(sourceDb, destDb, config);
|
|
503
|
-
|
|
504
|
-
// Validate transform with sample documents (in dry-run mode)
|
|
505
|
-
if (transformFn && config.dryRun) {
|
|
506
|
-
console.log('🧪 Validating transform with sample documents...');
|
|
507
|
-
let samplesTested = 0;
|
|
508
|
-
let samplesSkipped = 0;
|
|
509
|
-
let samplesErrors = 0;
|
|
510
|
-
|
|
511
|
-
for (const collection of config.collections) {
|
|
512
|
-
const snapshot = await sourceDb.collection(collection).limit(3).get();
|
|
513
|
-
for (const doc of snapshot.docs) {
|
|
514
|
-
try {
|
|
515
|
-
const result = transformFn(doc.data() as Record<string, unknown>, {
|
|
516
|
-
id: doc.id,
|
|
517
|
-
path: `${collection}/${doc.id}`,
|
|
518
|
-
});
|
|
519
|
-
if (result === null) {
|
|
520
|
-
samplesSkipped++;
|
|
521
|
-
} else {
|
|
522
|
-
samplesTested++;
|
|
523
|
-
}
|
|
524
|
-
} catch (error) {
|
|
525
|
-
samplesErrors++;
|
|
526
|
-
const err = error as Error;
|
|
527
|
-
console.error(` ⚠️ Transform error on ${collection}/${doc.id}: ${err.message}`);
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
if (samplesErrors > 0) {
|
|
533
|
-
console.log(` ❌ ${samplesErrors} sample(s) failed - review your transform function`);
|
|
534
|
-
} else if (samplesTested > 0 || samplesSkipped > 0) {
|
|
535
|
-
console.log(` ✓ Tested ${samplesTested} sample(s), ${samplesSkipped} would be skipped`);
|
|
536
|
-
}
|
|
537
|
-
console.log('');
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
if (!config.resume) {
|
|
541
|
-
stats = {
|
|
542
|
-
collectionsProcessed: 0,
|
|
543
|
-
documentsTransferred: 0,
|
|
544
|
-
documentsDeleted: 0,
|
|
545
|
-
errors: 0,
|
|
546
|
-
};
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
// Count total documents for progress bar
|
|
550
|
-
let totalDocs = 0;
|
|
551
|
-
let progressBar: cliProgress.SingleBar | null = null;
|
|
552
|
-
|
|
553
|
-
if (!argv.quiet) {
|
|
554
|
-
console.log('📊 Counting documents...');
|
|
555
|
-
let lastSubcollectionLog = Date.now();
|
|
556
|
-
let subcollectionCount = 0;
|
|
557
|
-
|
|
558
|
-
const countProgress: CountProgress = {
|
|
559
|
-
onCollection: (path, count) => {
|
|
560
|
-
console.log(` ${path}: ${count} documents`);
|
|
561
|
-
},
|
|
562
|
-
onSubcollection: (_path) => {
|
|
563
|
-
subcollectionCount++;
|
|
564
|
-
// Show progress every 2 seconds to avoid flooding the console
|
|
565
|
-
const now = Date.now();
|
|
566
|
-
if (now - lastSubcollectionLog > 2000) {
|
|
567
|
-
process.stdout.write(`\r Scanning subcollections... (${subcollectionCount} found)`);
|
|
568
|
-
lastSubcollectionLog = now;
|
|
569
|
-
}
|
|
570
|
-
},
|
|
571
|
-
};
|
|
572
|
-
|
|
573
|
-
for (const collection of config.collections) {
|
|
574
|
-
totalDocs += await countDocuments(sourceDb, collection, config, 0, countProgress);
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
// Clear the subcollection line if any were found
|
|
578
|
-
if (subcollectionCount > 0) {
|
|
579
|
-
process.stdout.write('\r' + ' '.repeat(60) + '\r');
|
|
580
|
-
console.log(` Subcollections scanned: ${subcollectionCount}`);
|
|
275
|
+
if (webhookValidation.warning) {
|
|
276
|
+
console.log(`\n⚠️ ${webhookValidation.warning}`);
|
|
581
277
|
}
|
|
582
|
-
console.log(` Total: ${totalDocs} documents to transfer\n`);
|
|
583
|
-
|
|
584
|
-
if (totalDocs > 0) {
|
|
585
|
-
progressBar = new cliProgress.SingleBar({
|
|
586
|
-
format: '📦 Progress |{bar}| {percentage}% | {value}/{total} docs | {speed} docs/s | ETA: {eta}s',
|
|
587
|
-
barCompleteChar: '█',
|
|
588
|
-
barIncompleteChar: '░',
|
|
589
|
-
hideCursor: true,
|
|
590
|
-
});
|
|
591
|
-
progressBar.start(totalDocs, 0, { speed: '0' });
|
|
592
|
-
|
|
593
|
-
// Track speed using transfer stats
|
|
594
|
-
let lastDocsTransferred = 0;
|
|
595
|
-
let lastTime = Date.now();
|
|
596
|
-
|
|
597
|
-
const speedInterval = setInterval(() => {
|
|
598
|
-
if (progressBar) {
|
|
599
|
-
const now = Date.now();
|
|
600
|
-
const timeDiff = (now - lastTime) / 1000;
|
|
601
|
-
const currentDocs = stats.documentsTransferred;
|
|
602
|
-
|
|
603
|
-
if (timeDiff > 0) {
|
|
604
|
-
const docsDiff = currentDocs - lastDocsTransferred;
|
|
605
|
-
const speed = Math.round(docsDiff / timeDiff);
|
|
606
|
-
lastDocsTransferred = currentDocs;
|
|
607
|
-
lastTime = now;
|
|
608
|
-
progressBar.update({ speed: String(speed) });
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
}, 500);
|
|
612
|
-
|
|
613
|
-
// Store interval for cleanup
|
|
614
|
-
(progressBar as unknown as { _speedInterval: NodeJS.Timeout })._speedInterval = speedInterval;
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
// Clear destination collections if enabled
|
|
619
|
-
if (config.clear) {
|
|
620
|
-
console.log('🗑️ Clearing destination collections...');
|
|
621
|
-
for (const collection of config.collections) {
|
|
622
|
-
const destCollection = getDestCollectionPath(collection, config.renameCollection);
|
|
623
|
-
const deleted = await clearCollection(
|
|
624
|
-
destDb,
|
|
625
|
-
destCollection,
|
|
626
|
-
config,
|
|
627
|
-
logger,
|
|
628
|
-
config.includeSubcollections
|
|
629
|
-
);
|
|
630
|
-
stats.documentsDeleted += deleted;
|
|
631
|
-
}
|
|
632
|
-
console.log(` Deleted ${stats.documentsDeleted} documents\n`);
|
|
633
278
|
}
|
|
634
279
|
|
|
635
|
-
//
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
280
|
+
// Exit early if only validating config (for testing)
|
|
281
|
+
if (argv.validateOnly) {
|
|
282
|
+
console.log('\n✓ Configuration is valid');
|
|
283
|
+
process.exit(0);
|
|
639
284
|
}
|
|
640
285
|
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
);
|
|
648
|
-
if (errors.length > 0) {
|
|
649
|
-
for (const err of errors) {
|
|
650
|
-
logger.error('Parallel transfer error', { error: err.message });
|
|
651
|
-
stats.errors++;
|
|
652
|
-
}
|
|
653
|
-
}
|
|
654
|
-
} else {
|
|
655
|
-
for (const collection of config.collections) {
|
|
656
|
-
try {
|
|
657
|
-
await transferCollection(ctx, collection);
|
|
658
|
-
} catch (error) {
|
|
659
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
660
|
-
logger.error(`Transfer failed for ${collection}`, { error: err.message });
|
|
661
|
-
stats.errors++;
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
if (progressBar) {
|
|
667
|
-
// Clear speed update interval
|
|
668
|
-
const interval = (progressBar as unknown as { _speedInterval?: NodeJS.Timeout })._speedInterval;
|
|
669
|
-
if (interval) {
|
|
670
|
-
clearInterval(interval);
|
|
671
|
-
}
|
|
672
|
-
progressBar.stop();
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
// Delete orphan documents if enabled (sync mode)
|
|
676
|
-
if (config.deleteMissing) {
|
|
677
|
-
console.log('\n🔄 Deleting orphan documents (sync mode)...');
|
|
678
|
-
for (const collection of config.collections) {
|
|
679
|
-
const deleted = await deleteOrphanDocuments(
|
|
680
|
-
sourceDb,
|
|
681
|
-
destDb,
|
|
682
|
-
collection,
|
|
683
|
-
config,
|
|
684
|
-
logger
|
|
685
|
-
);
|
|
686
|
-
stats.documentsDeleted += deleted;
|
|
687
|
-
}
|
|
688
|
-
if (stats.documentsDeleted > 0) {
|
|
689
|
-
console.log(` Deleted ${stats.documentsDeleted} orphan documents`);
|
|
690
|
-
} else {
|
|
691
|
-
console.log(' No orphan documents found');
|
|
286
|
+
// Skip confirmation in interactive mode (already confirmed by selection)
|
|
287
|
+
if (!argv.yes && !argv.interactive) {
|
|
288
|
+
const confirmed = await askConfirmation(config);
|
|
289
|
+
if (!confirmed) {
|
|
290
|
+
console.log('\n🚫 Transfer cancelled by user\n');
|
|
291
|
+
process.exit(0);
|
|
692
292
|
}
|
|
693
293
|
}
|
|
694
294
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
295
|
+
// Initialize output
|
|
296
|
+
const output = new Output({
|
|
297
|
+
quiet: argv.quiet,
|
|
298
|
+
json: argv.json,
|
|
299
|
+
logFile: argv.log,
|
|
300
|
+
maxLogSize: parseSize(argv.maxLogSize),
|
|
700
301
|
});
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
// Verify transfer if enabled (and not dry-run)
|
|
704
|
-
let verifyResult: Record<string, { source: number; dest: number; match: boolean }> | null = null;
|
|
705
|
-
if (config.verify && !config.dryRun) {
|
|
706
|
-
if (!config.json) {
|
|
707
|
-
console.log('\n🔍 Verifying transfer...');
|
|
708
|
-
}
|
|
709
|
-
verifyResult = {};
|
|
710
|
-
let verifyPassed = true;
|
|
711
|
-
|
|
712
|
-
for (const collection of config.collections) {
|
|
713
|
-
const destCollection = getDestCollectionPath(collection, config.renameCollection);
|
|
302
|
+
output.init();
|
|
303
|
+
output.logInfo('Transfer started', { config: config as unknown as Record<string, unknown> });
|
|
714
304
|
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
const sourceTotal = sourceCount.data().count;
|
|
305
|
+
// Run transfer
|
|
306
|
+
const result = await runTransfer(config, argv, output);
|
|
718
307
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
const destTotal = destCount.data().count;
|
|
722
|
-
|
|
723
|
-
const match = sourceTotal === destTotal;
|
|
724
|
-
verifyResult[collection] = { source: sourceTotal, dest: destTotal, match };
|
|
725
|
-
|
|
726
|
-
if (!config.json) {
|
|
727
|
-
if (match) {
|
|
728
|
-
console.log(` ✓ ${collection}: ${sourceTotal} docs (matched)`);
|
|
729
|
-
} else {
|
|
730
|
-
console.log(` ⚠️ ${collection}: source=${sourceTotal}, dest=${destTotal} (mismatch)`);
|
|
731
|
-
}
|
|
732
|
-
}
|
|
733
|
-
if (!match) verifyPassed = false;
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
if (!config.json) {
|
|
737
|
-
if (verifyPassed) {
|
|
738
|
-
console.log(' ✓ Verification passed');
|
|
739
|
-
} else {
|
|
740
|
-
console.log(' ⚠️ Verification found mismatches');
|
|
741
|
-
}
|
|
742
|
-
}
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
// Delete state file on successful completion (before JSON output)
|
|
746
|
-
if (!config.dryRun) {
|
|
747
|
-
deleteTransferState(config.stateFile);
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
// JSON output mode
|
|
751
|
-
if (config.json) {
|
|
752
|
-
const jsonOutput = {
|
|
753
|
-
success: true,
|
|
754
|
-
dryRun: config.dryRun,
|
|
755
|
-
source: config.sourceProject,
|
|
756
|
-
destination: config.destProject,
|
|
757
|
-
collections: config.collections,
|
|
758
|
-
stats: {
|
|
759
|
-
collectionsProcessed: stats.collectionsProcessed,
|
|
760
|
-
documentsTransferred: stats.documentsTransferred,
|
|
761
|
-
documentsDeleted: stats.documentsDeleted,
|
|
762
|
-
errors: stats.errors,
|
|
763
|
-
},
|
|
764
|
-
duration: Number.parseFloat(duration),
|
|
765
|
-
verify: verifyResult,
|
|
766
|
-
};
|
|
767
|
-
console.log(JSON.stringify(jsonOutput, null, 2));
|
|
768
|
-
} else {
|
|
769
|
-
console.log('\n' + '='.repeat(60));
|
|
770
|
-
console.log('📊 TRANSFER SUMMARY');
|
|
771
|
-
console.log('='.repeat(60));
|
|
772
|
-
console.log(`Collections processed: ${stats.collectionsProcessed}`);
|
|
773
|
-
if (stats.documentsDeleted > 0) {
|
|
774
|
-
console.log(`Documents deleted: ${stats.documentsDeleted}`);
|
|
775
|
-
}
|
|
776
|
-
console.log(`Documents transferred: ${stats.documentsTransferred}`);
|
|
777
|
-
console.log(`Errors: ${stats.errors}`);
|
|
778
|
-
console.log(`Duration: ${duration}s`);
|
|
779
|
-
|
|
780
|
-
if (argv.log) {
|
|
781
|
-
console.log(`Log file: ${argv.log}`);
|
|
782
|
-
}
|
|
783
|
-
|
|
784
|
-
if (config.dryRun) {
|
|
785
|
-
console.log('\n⚠ DRY RUN: No data was actually written');
|
|
786
|
-
console.log(' Run with --dry-run=false to perform the transfer');
|
|
787
|
-
} else {
|
|
788
|
-
console.log('\n✓ Transfer completed successfully');
|
|
789
|
-
}
|
|
790
|
-
console.log('='.repeat(60) + '\n');
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
// Send webhook notification if configured
|
|
794
|
-
if (config.webhook) {
|
|
795
|
-
await sendWebhook(
|
|
796
|
-
config.webhook,
|
|
797
|
-
{
|
|
798
|
-
source: config.sourceProject!,
|
|
799
|
-
destination: config.destProject!,
|
|
800
|
-
collections: config.collections,
|
|
801
|
-
stats,
|
|
802
|
-
duration: Number.parseFloat(duration),
|
|
803
|
-
dryRun: config.dryRun,
|
|
804
|
-
success: true,
|
|
805
|
-
},
|
|
806
|
-
logger
|
|
807
|
-
);
|
|
308
|
+
if (!result.success) {
|
|
309
|
+
process.exit(1);
|
|
808
310
|
}
|
|
311
|
+
}
|
|
809
312
|
|
|
810
|
-
|
|
313
|
+
// Run main
|
|
314
|
+
try {
|
|
315
|
+
await main();
|
|
811
316
|
} catch (error) {
|
|
812
|
-
|
|
813
|
-
const duration = Number.parseFloat(((Date.now() - startTime) / 1000).toFixed(2));
|
|
814
|
-
|
|
815
|
-
// JSON output mode for errors
|
|
816
|
-
if (config.json) {
|
|
817
|
-
const jsonOutput = {
|
|
818
|
-
success: false,
|
|
819
|
-
error: errorMessage,
|
|
820
|
-
dryRun: config.dryRun,
|
|
821
|
-
source: config.sourceProject,
|
|
822
|
-
destination: config.destProject,
|
|
823
|
-
collections: config.collections,
|
|
824
|
-
stats: {
|
|
825
|
-
collectionsProcessed: stats.collectionsProcessed,
|
|
826
|
-
documentsTransferred: stats.documentsTransferred,
|
|
827
|
-
documentsDeleted: stats.documentsDeleted,
|
|
828
|
-
errors: stats.errors,
|
|
829
|
-
},
|
|
830
|
-
duration,
|
|
831
|
-
};
|
|
832
|
-
console.log(JSON.stringify(jsonOutput, null, 2));
|
|
833
|
-
} else {
|
|
834
|
-
console.error('\n❌ Error during transfer:', errorMessage);
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
// Send webhook notification on error if configured
|
|
838
|
-
if (config.webhook && logger) {
|
|
839
|
-
await sendWebhook(
|
|
840
|
-
config.webhook,
|
|
841
|
-
{
|
|
842
|
-
source: config.sourceProject ?? 'unknown',
|
|
843
|
-
destination: config.destProject ?? 'unknown',
|
|
844
|
-
collections: config.collections,
|
|
845
|
-
stats,
|
|
846
|
-
duration,
|
|
847
|
-
dryRun: config.dryRun,
|
|
848
|
-
success: false,
|
|
849
|
-
error: errorMessage,
|
|
850
|
-
},
|
|
851
|
-
logger
|
|
852
|
-
);
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
await cleanupFirebase();
|
|
317
|
+
console.error('\n❌ Unexpected error:', (error as Error).message);
|
|
856
318
|
process.exit(1);
|
|
857
319
|
}
|