@fazetitans/fscopy 1.0.1 → 1.1.1

package/src/cli.ts

@@ -1,116 +1,31 @@
 #!/usr/bin/env bun
 
+// Suppress GCE metadata lookup warning (we're not running on Google Cloud)
+process.env.METADATA_SERVER_DETECTION = 'none';
+
 import admin from 'firebase-admin';
-import type { Firestore, DocumentReference, WriteBatch } from 'firebase-admin/firestore';
+import type { Firestore } from 'firebase-admin/firestore';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import ini from 'ini';
 import cliProgress from 'cli-progress';
 import fs from 'node:fs';
 import path from 'node:path';
 import readline from 'node:readline';
-import { input, confirm, checkbox } from '@inquirer/prompts';
-
-// =============================================================================
-// Types
-// =============================================================================
-
-interface WhereFilter {
-    field: string;
-    operator: FirebaseFirestore.WhereFilterOp;
-    value: string | number | boolean;
-}
-
-interface Config {
-    collections: string[];
-    includeSubcollections: boolean;
-    dryRun: boolean;
-    batchSize: number;
-    limit: number;
-    sourceProject: string | null;
-    destProject: string | null;
-    retries: number;
-    where: WhereFilter[];
-    exclude: string[];
-    merge: boolean;
-    parallel: number;
-    clear: boolean;
-    deleteMissing: boolean;
-    transform: string | null;
-    renameCollection: Record<string, string>;
-    idPrefix: string | null;
-    idSuffix: string | null;
-    webhook: string | null;
-    resume: boolean;
-    stateFile: string;
-}
 
-type TransformFunction = (
-    doc: Record<string, unknown>,
-    meta: { id: string; path: string }
-) => Record<string, unknown> | null;
-
-interface Stats {
-    collectionsProcessed: number;
-    documentsTransferred: number;
-    documentsDeleted: number;
-    errors: number;
-}
-
-interface TransferState {
-    version: number;
-    sourceProject: string;
-    destProject: string;
-    collections: string[];
-    startedAt: string;
-    updatedAt: string;
-    completedDocs: Record<string, string[]>; // collectionPath -> array of doc IDs
-    stats: Stats;
-}
-
-interface LogEntry {
-    timestamp: string;
-    level: string;
-    message: string;
-    [key: string]: unknown;
-}
-
-interface RetryOptions {
-    retries?: number;
-    baseDelay?: number;
-    maxDelay?: number;
-    onRetry?: (attempt: number, max: number, error: Error, delay: number) => void;
-}
-
-interface CliArgs {
-    init?: string;
-    config?: string;
-    collections?: string[];
-    includeSubcollections?: boolean;
-    dryRun?: boolean;
-    batchSize?: number;
-    limit?: number;
-    sourceProject?: string;
-    destProject?: string;
-    yes: boolean;
-    log?: string;
-    retries: number;
-    quiet: boolean;
-    where?: string[];
-    exclude?: string[];
-    merge?: boolean;
-    parallel?: number;
-    clear?: boolean;
-    deleteMissing?: boolean;
-    interactive?: boolean;
-    transform?: string;
-    renameCollection?: string[];
-    idPrefix?: string;
-    idSuffix?: string;
-    webhook?: string;
-    resume?: boolean;
-    stateFile?: string;
-}
+// Import from modules
+import type { Config, Stats, TransferState, TransformFunction, CliArgs } from './types.js';
+import { Logger } from './utils/logger.js';
+import { ensureCredentials } from './utils/credentials.js';
+import { formatFirebaseError } from './utils/errors.js';
+import { RateLimiter } from './utils/rate-limiter.js';
+import { loadConfigFile, mergeConfig } from './config/parser.js';
+import { validateConfig } from './config/validator.js';
+import { defaults } from './config/defaults.js';
+import { generateConfigFile } from './config/generator.js';
+import { loadTransferState, saveTransferState, createInitialState, validateStateForResume, deleteTransferState } from './state/index.js';
+import { sendWebhook } from './webhook/index.js';
+import { countDocuments, transferCollection, clearCollection, deleteOrphanDocuments, processInParallel, getDestCollectionPath, type TransferContext, type CountProgress } from './transfer/index.js';
+import { runInteractiveMode } from './interactive.js';
 
 // =============================================================================
 // CLI Arguments
@@ -254,6 +169,26 @@ const argv = yargs(hideBin(process.argv))
         description: 'Path to state file for resume (default: .fscopy-state.json)',
         default: '.fscopy-state.json',
     })
+    .option('verify', {
+        type: 'boolean',
+        description: 'Verify document counts after transfer',
+        default: false,
+    })
+    .option('rate-limit', {
+        type: 'number',
+        description: 'Limit transfer rate (documents per second, 0 = unlimited)',
+        default: 0,
+    })
+    .option('skip-oversized', {
+        type: 'boolean',
+        description: 'Skip documents exceeding 1MB instead of failing',
+        default: false,
+    })
+    .option('json', {
+        type: 'boolean',
+        description: 'Output results in JSON format (for CI/CD)',
+        default: false,
+    })
     .example('$0 --init config.ini', 'Generate INI config template (default)')
     .example('$0 --init config.json', 'Generate JSON config template')
     .example('$0 -f config.ini', 'Run transfer with config file')
@@ -274,508 +209,6 @@ const argv = yargs(hideBin(process.argv))
     .help()
     .parseSync() as CliArgs;
 
-// =============================================================================
-// Constants & Templates
-// =============================================================================
-
-const defaults: Config = {
-    collections: [],
-    includeSubcollections: false,
-    dryRun: true,
-    batchSize: 500,
-    limit: 0,
-    sourceProject: null,
-    destProject: null,
-    retries: 3,
-    where: [],
-    exclude: [],
-    merge: false,
-    parallel: 1,
-    clear: false,
-    deleteMissing: false,
-    transform: null,
-    renameCollection: {},
-    idPrefix: null,
-    idSuffix: null,
-    webhook: null,
-    resume: false,
-    stateFile: '.fscopy-state.json',
-};
-
-const iniTemplate = `; fscopy configuration file
-
-[projects]
-source = my-source-project
-dest = my-dest-project
-
-[transfer]
-; Comma-separated list of collections
-collections = collection1, collection2
-includeSubcollections = false
-dryRun = true
-batchSize = 500
-limit = 0
-
-[options]
-; Filter documents: "field operator value" (operators: ==, !=, <, >, <=, >=)
-; where = status == active
-; Exclude subcollections by pattern (comma-separated, supports glob)
-; exclude = logs, temp/*, cache
-; Merge documents instead of overwriting
-merge = false
-; Number of parallel collection transfers
-parallel = 1
-; Clear destination collections before transfer (DESTRUCTIVE)
-clear = false
-; Delete destination docs not present in source (sync mode)
-deleteMissing = false
-; Transform documents during transfer (path to JS/TS file)
-; transform = ./transforms/anonymize.ts
-; Rename collections in destination (format: source:dest, comma-separated)
-; renameCollection = users:users_backup, orders:orders_2024
-; Add prefix or suffix to document IDs
-; idPrefix = backup_
-; idSuffix = _v2
-; Webhook URL for transfer notifications (Slack, Discord, or custom)
-; webhook = https://hooks.slack.com/services/...
-`;
-
-const jsonTemplate = {
-    sourceProject: 'my-source-project',
-    destProject: 'my-dest-project',
-    collections: ['collection1', 'collection2'],
-    includeSubcollections: false,
-    dryRun: true,
-    batchSize: 500,
-    limit: 0,
-    where: [],
-    exclude: [],
-    merge: false,
-    parallel: 1,
-    clear: false,
-    deleteMissing: false,
-    transform: null,
-    renameCollection: {},
-    idPrefix: null,
-    idSuffix: null,
-    webhook: null,
-};
-
-// =============================================================================
-// Logger
-// =============================================================================
-
-class Logger {
-    private readonly logPath: string | undefined;
-    private readonly entries: LogEntry[] = [];
-    private readonly startTime: Date;
-
-    constructor(logPath?: string) {
-        this.logPath = logPath;
-        this.startTime = new Date();
-    }
-
-    log(level: string, message: string, data: Record<string, unknown> = {}): void {
-        const entry: LogEntry = {
-            timestamp: new Date().toISOString(),
-            level,
-            message,
-            ...data,
-        };
-        this.entries.push(entry);
-
-        if (this.logPath) {
-            const line =
-                `[${entry.timestamp}] [${level}] ${message}` +
-                (Object.keys(data).length > 0 ? ` ${JSON.stringify(data)}` : '') +
-                '\n';
-            fs.appendFileSync(this.logPath, line);
-        }
-    }
-
-    info(message: string, data?: Record<string, unknown>): void {
-        this.log('INFO', message, data);
-    }
-
-    error(message: string, data?: Record<string, unknown>): void {
-        this.log('ERROR', message, data);
-    }
-
-    success(message: string, data?: Record<string, unknown>): void {
-        this.log('SUCCESS', message, data);
-    }
-
-    init(): void {
-        if (this.logPath) {
-            const header = `# fscopy transfer log\n# Started: ${this.startTime.toISOString()}\n\n`;
-            fs.writeFileSync(this.logPath, header);
-        }
-    }
-
-    summary(stats: Stats, duration: string): void {
-        if (this.logPath) {
-            let summary = `\n# Summary\n# Collections: ${stats.collectionsProcessed}\n`;
-            if (stats.documentsDeleted > 0) {
-                summary += `# Deleted: ${stats.documentsDeleted}\n`;
-            }
-            summary += `# Transferred: ${stats.documentsTransferred}\n# Errors: ${stats.errors}\n# Duration: ${duration}s\n`;
-            fs.appendFileSync(this.logPath, summary);
-        }
-    }
-}
-
-// =============================================================================
-// Retry Logic
-// =============================================================================
-
-async function withRetry<T>(fn: () => Promise<T>, options: RetryOptions = {}): Promise<T> {
-    const { retries = 3, baseDelay = 1000, maxDelay = 30000, onRetry } = options;
-
-    let lastError: Error | undefined;
-    for (let attempt = 0; attempt <= retries; attempt++) {
-        try {
-            return await fn();
-        } catch (error) {
-            lastError = error as Error;
-
-            if (attempt < retries) {
-                const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
-                if (onRetry) {
-                    onRetry(attempt + 1, retries, lastError, delay);
-                }
-                await new Promise((resolve) => setTimeout(resolve, delay));
-            }
-        }
-    }
-
-    throw lastError;
-}
-
-// =============================================================================
-// Config Parsing
-// =============================================================================
-
-function getFileFormat(filePath: string): 'json' | 'ini' {
-    const ext = path.extname(filePath).toLowerCase();
-    if (ext === '.json') return 'json';
-    return 'ini';
-}
-
-function parseBoolean(val: unknown): boolean {
-    if (typeof val === 'boolean') return val;
-    if (typeof val === 'string') {
-        return val.toLowerCase() === 'true';
-    }
-    return false;
-}
-
-function parseWhereFilter(filterStr: string): WhereFilter | null {
-    // Parse "field operator value" format
-    const operatorRegex = /(==|!=|<=|>=|<|>)/;
-    const match = new RegExp(operatorRegex).exec(filterStr);
-
-    if (!match) {
-        console.warn(`⚠️  Invalid where filter: "${filterStr}" (missing operator)`);
-        return null;
-    }
-
-    const operator = match[0] as FirebaseFirestore.WhereFilterOp;
-    const [fieldPart, valuePart] = filterStr.split(operatorRegex).filter((_, i) => i !== 1);
-
-    if (!fieldPart || !valuePart) {
-        console.warn(`⚠️  Invalid where filter: "${filterStr}" (missing field or value)`);
-        return null;
-    }
-
-    const field = fieldPart.trim();
-    const rawValue = valuePart.trim();
-
-    // Parse value type
-    let value: string | number | boolean;
-    if (rawValue === 'true') {
-        value = true;
-    } else if (rawValue === 'false') {
-        value = false;
-    } else if (rawValue === 'null') {
-        value = null as unknown as string; // Firestore supports null
-    } else if (!Number.isNaN(Number(rawValue)) && rawValue !== '') {
-        value = Number(rawValue);
-    } else {
-        // Remove quotes if present
-        value = rawValue.replaceAll(/(?:^["'])|(?:["']$)/g, '');
-    }
-
-    return { field, operator, value };
-}
-
-function parseWhereFilters(filters: string[] | undefined): WhereFilter[] {
-    if (!filters || filters.length === 0) return [];
-    return filters.map(parseWhereFilter).filter((f): f is WhereFilter => f !== null);
-}
-
-function parseStringList(value: string | undefined): string[] {
-    if (!value) return [];
-    return value
-        .split(',')
-        .map((s) => s.trim())
-        .filter((s) => s.length > 0);
-}
-
-function parseRenameMapping(mappings: string[] | string | undefined): Record<string, string> {
-    if (!mappings) return {};
-
-    const result: Record<string, string> = {};
-    const items = Array.isArray(mappings) ? mappings : parseStringList(mappings);
-
-    for (const item of items) {
-        const mapping = String(item).trim();
-        const colonIndex = mapping.indexOf(':');
-        if (colonIndex === -1) {
-            console.warn(`⚠️  Invalid rename mapping: "${mapping}" (missing ':')`);
-            continue;
-        }
-        const source = mapping.slice(0, colonIndex).trim();
-        const dest = mapping.slice(colonIndex + 1).trim();
-        if (!source || !dest) {
-            console.warn(`⚠️  Invalid rename mapping: "${mapping}" (empty source or dest)`);
-            continue;
-        }
-        result[source] = dest;
-    }
-
-    return result;
-}
-
-function matchesExcludePattern(path: string, patterns: string[]): boolean {
-    for (const pattern of patterns) {
-        if (pattern.includes('*')) {
-            // Convert glob pattern to regex
-            const regex = new RegExp('^' + pattern.replaceAll('*', '.*') + '$');
-            if (regex.test(path)) return true;
-        } else if (path === pattern || path.endsWith('/' + pattern)) {
-            // Exact match or ends with pattern
-            return true;
-        }
-    }
-    return false;
-}
-
-function parseIniConfig(content: string): Partial<Config> {
-    const parsed = ini.parse(content) as {
-        projects?: { source?: string; dest?: string };
-        transfer?: {
-            collections?: string;
-            includeSubcollections?: string | boolean;
-            dryRun?: string | boolean;
-            batchSize?: string;
-            limit?: string;
-        };
-        options?: {
-            where?: string;
-            exclude?: string;
-            merge?: string | boolean;
-            parallel?: string;
-            clear?: string | boolean;
-            deleteMissing?: string | boolean;
-            transform?: string;
-            renameCollection?: string;
-            idPrefix?: string;
-            idSuffix?: string;
-            webhook?: string;
-        };
-    };
-
-    let collections: string[] = [];
-    if (parsed.transfer?.collections) {
-        collections = parsed.transfer.collections
-            .split(',')
-            .map((c) => c.trim())
-            .filter((c) => c.length > 0);
-    }
-
-    // Parse where filters from INI (single filter per line or comma-separated)
-    const whereFilters = parsed.options?.where
-        ? parseWhereFilters(parseStringList(parsed.options.where))
-        : [];
-
-    return {
-        sourceProject: parsed.projects?.source ?? null,
-        destProject: parsed.projects?.dest ?? null,
-        collections,
-        includeSubcollections: parseBoolean(parsed.transfer?.includeSubcollections),
-        dryRun: parseBoolean(parsed.transfer?.dryRun ?? 'true'),
-        batchSize: Number.parseInt(parsed.transfer?.batchSize ?? '', 10) || 500,
-        limit: Number.parseInt(parsed.transfer?.limit ?? '', 10) || 0,
-        where: whereFilters,
-        exclude: parseStringList(parsed.options?.exclude),
-        merge: parseBoolean(parsed.options?.merge),
-        parallel: Number.parseInt(parsed.options?.parallel ?? '', 10) || 1,
-        clear: parseBoolean(parsed.options?.clear),
-        deleteMissing: parseBoolean(parsed.options?.deleteMissing),
-        transform: parsed.options?.transform ?? null,
-        renameCollection: parseRenameMapping(parsed.options?.renameCollection),
-        idPrefix: parsed.options?.idPrefix ?? null,
-        idSuffix: parsed.options?.idSuffix ?? null,
-        webhook: parsed.options?.webhook ?? null,
-    };
-}
-
-function parseJsonConfig(content: string): Partial<Config> {
-    const config = JSON.parse(content) as {
-        sourceProject?: string;
-        destProject?: string;
-        collections?: string[];
-        includeSubcollections?: boolean;
-        dryRun?: boolean;
-        batchSize?: number;
-        limit?: number;
-        where?: string[];
-        exclude?: string[];
-        merge?: boolean;
-        parallel?: number;
-        clear?: boolean;
-        deleteMissing?: boolean;
-        transform?: string;
-        renameCollection?: Record<string, string>;
-        idPrefix?: string;
-        idSuffix?: string;
-        webhook?: string;
-    };
-
-    return {
-        sourceProject: config.sourceProject ?? null,
-        destProject: config.destProject ?? null,
-        collections: config.collections,
-        includeSubcollections: config.includeSubcollections,
-        dryRun: config.dryRun,
-        batchSize: config.batchSize,
-        limit: config.limit,
-        where: parseWhereFilters(config.where),
-        exclude: config.exclude,
-        merge: config.merge,
-        parallel: config.parallel,
-        clear: config.clear,
-        deleteMissing: config.deleteMissing,
-        transform: config.transform ?? null,
-        renameCollection: config.renameCollection ?? {},
-        idPrefix: config.idPrefix ?? null,
-        idSuffix: config.idSuffix ?? null,
-        webhook: config.webhook ?? null,
-    };
-}
-
-function loadConfigFile(configPath?: string): Partial<Config> {
-    if (!configPath) return {};
-
-    const absolutePath = path.resolve(configPath);
-    if (!fs.existsSync(absolutePath)) {
-        throw new Error(`Config file not found: ${absolutePath}`);
-    }
-
-    const content = fs.readFileSync(absolutePath, 'utf-8');
-    const format = getFileFormat(absolutePath);
-
-    console.log(`📄 Loaded config from: ${absolutePath} (${format.toUpperCase()})\n`);
-
-    return format === 'json' ? parseJsonConfig(content) : parseIniConfig(content);
-}
-
-function mergeConfig(defaultConfig: Config, fileConfig: Partial<Config>, cliArgs: CliArgs): Config {
-    // Parse CLI where filters
-    const cliWhereFilters = parseWhereFilters(cliArgs.where);
-
-    // Parse CLI rename collection mappings
-    const cliRenameCollection = parseRenameMapping(cliArgs.renameCollection);
-
-    return {
-        collections: cliArgs.collections ?? fileConfig.collections ?? defaultConfig.collections,
-        includeSubcollections:
-            cliArgs.includeSubcollections ??
-            fileConfig.includeSubcollections ??
-            defaultConfig.includeSubcollections,
-        dryRun: cliArgs.dryRun ?? fileConfig.dryRun ?? defaultConfig.dryRun,
-        batchSize: cliArgs.batchSize ?? fileConfig.batchSize ?? defaultConfig.batchSize,
-        limit: cliArgs.limit ?? fileConfig.limit ?? defaultConfig.limit,
-        sourceProject:
-            cliArgs.sourceProject ?? fileConfig.sourceProject ?? defaultConfig.sourceProject,
-        destProject: cliArgs.destProject ?? fileConfig.destProject ?? defaultConfig.destProject,
-        retries: cliArgs.retries ?? defaultConfig.retries,
-        where:
-            cliWhereFilters.length > 0
-                ? cliWhereFilters
-                : (fileConfig.where ?? defaultConfig.where),
-        exclude: cliArgs.exclude ?? fileConfig.exclude ?? defaultConfig.exclude,
-        merge: cliArgs.merge ?? fileConfig.merge ?? defaultConfig.merge,
-        parallel: cliArgs.parallel ?? fileConfig.parallel ?? defaultConfig.parallel,
-        clear: cliArgs.clear ?? fileConfig.clear ?? defaultConfig.clear,
-        deleteMissing: cliArgs.deleteMissing ?? fileConfig.deleteMissing ?? defaultConfig.deleteMissing,
-        transform: cliArgs.transform ?? fileConfig.transform ?? defaultConfig.transform,
-        renameCollection:
-            Object.keys(cliRenameCollection).length > 0
-                ? cliRenameCollection
-                : (fileConfig.renameCollection ?? defaultConfig.renameCollection),
-        idPrefix: cliArgs.idPrefix ?? fileConfig.idPrefix ?? defaultConfig.idPrefix,
-        idSuffix: cliArgs.idSuffix ?? fileConfig.idSuffix ?? defaultConfig.idSuffix,
-        webhook: cliArgs.webhook ?? fileConfig.webhook ?? defaultConfig.webhook,
-        resume: cliArgs.resume ?? defaultConfig.resume,
-        stateFile: cliArgs.stateFile ?? defaultConfig.stateFile,
-    };
-}
-
-function validateConfig(config: Config): string[] {
-    const errors: string[] = [];
-
-    if (!config.sourceProject) {
-        errors.push('Source project is required (--source-project or in config file)');
-    }
-    if (!config.destProject) {
-        errors.push('Destination project is required (--dest-project or in config file)');
-    }
-    if (config.sourceProject && config.destProject && config.sourceProject === config.destProject) {
-        // Same project is allowed only if we're renaming collections or modifying IDs
-        const hasRenamedCollections = Object.keys(config.renameCollection).length > 0;
-        const hasIdModification = config.idPrefix !== null || config.idSuffix !== null;
-
-        if (!hasRenamedCollections && !hasIdModification) {
-            errors.push(
-                'Source and destination projects are the same. ' +
-                'Use --rename-collection or --id-prefix/--id-suffix to avoid overwriting data.'
-            );
-        }
-    }
-    if (!config.collections || config.collections.length === 0) {
-        errors.push('At least one collection is required (-c or --collections)');
-    }
-
-    return errors;
-}
-
-// =============================================================================
-// Config File Generation
-// =============================================================================
-
-function generateConfigFile(outputPath: string): boolean {
-    const filePath = path.resolve(outputPath);
-    const format = getFileFormat(filePath);
-
-    if (fs.existsSync(filePath)) {
-        console.error(`❌ File already exists: ${filePath}`);
-        console.error('   Use a different filename or delete the existing file.');
-        process.exitCode = 1;
-        return false;
-    }
-
-    const content = format === 'json' ? JSON.stringify(jsonTemplate, null, 4) : iniTemplate;
-
-    fs.writeFileSync(filePath, content, 'utf-8');
-
-    console.log(`✓ Config template created: ${filePath}`);
-    console.log('');
-    console.log('Edit the file to configure your transfer, then run:');
-    console.log(`  fscopy -f ${outputPath}`);
-
-    return true;
-}
 
 // =============================================================================
 // Transform Loading
@@ -867,6 +300,12 @@ function displayConfig(config: Config): void {
             .join(', ');
         console.log(`  🏷️  ID modification:      ${idMod}`);
     }
+    if (config.rateLimit > 0) {
+        console.log(`  ⏱️  Rate limit:          ${config.rateLimit} docs/s`);
+    }
+    if (config.skipOversized) {
+        console.log(`  📏 Skip oversized:       enabled (skip docs > 1MB)`);
+    }
 
     console.log('');
 
@@ -895,116 +334,6 @@ async function askConfirmation(config: Config): Promise<boolean> {
     });
 }
 
-// =============================================================================
-// Interactive Mode
-// =============================================================================
-
-async function runInteractiveMode(config: Config): Promise<Config> {
-    console.log('\n' + '='.repeat(60));
-    console.log('🔄 FSCOPY - INTERACTIVE MODE');
-    console.log('='.repeat(60) + '\n');
-
-    // Prompt for source project if not set
-    let sourceProject = config.sourceProject;
-    if (!sourceProject) {
-        sourceProject = await input({
-            message: 'Source Firebase project ID:',
-            validate: (value) => value.length > 0 || 'Project ID is required',
-        });
-    } else {
-        console.log(`📤 Source project: ${sourceProject}`);
-    }
-
-    // Prompt for destination project if not set
-    let destProject = config.destProject;
-    if (!destProject) {
-        destProject = await input({
-            message: 'Destination Firebase project ID:',
-            validate: (value) => {
-                if (value.length === 0) return 'Project ID is required';
-                if (value === sourceProject) return 'Must be different from source';
-                return true;
-            },
-        });
-    } else {
-        console.log(`📥 Destination project: ${destProject}`);
-    }
-
-    // Initialize source Firebase to list collections
-    console.log('\n📊 Connecting to source project...');
-    const tempSourceApp = admin.initializeApp(
-        {
-            credential: admin.credential.applicationDefault(),
-            projectId: sourceProject,
-        },
-        'interactive-source'
-    );
-    const sourceDb = tempSourceApp.firestore();
-
-    // List all root collections
-    const rootCollections = await sourceDb.listCollections();
-    const collectionIds = rootCollections.map((col) => col.id);
-
-    if (collectionIds.length === 0) {
-        console.log('\n⚠️  No collections found in source project');
-        await tempSourceApp.delete();
-        process.exit(0);
-    }
-
-    // Count documents in each collection for preview
-    console.log('\n📋 Available collections:');
-    const collectionInfo: { id: string; count: number }[] = [];
-    for (const id of collectionIds) {
-        const snapshot = await sourceDb.collection(id).count().get();
-        const count = snapshot.data().count;
-        collectionInfo.push({ id, count });
-        console.log(`   - ${id} (${count} documents)`);
-    }
-
-    // Let user select collections
-    console.log('');
-    const selectedCollections = await checkbox({
-        message: 'Select collections to transfer:',
-        choices: collectionInfo.map((col) => ({
-            name: `${col.id} (${col.count} docs)`,
-            value: col.id,
-            checked: config.collections.includes(col.id),
-        })),
-        validate: (value) => value.length > 0 || 'Select at least one collection',
-    });
-
-    // Ask about options
-    console.log('');
-    const includeSubcollections = await confirm({
-        message: 'Include subcollections?',
-        default: config.includeSubcollections,
-    });
-
-    const dryRun = await confirm({
-        message: 'Dry run mode (preview without writing)?',
-        default: config.dryRun,
-    });
-
-    const merge = await confirm({
-        message: 'Merge mode (update instead of overwrite)?',
-        default: config.merge,
-    });
-
-    // Clean up temporary app
-    await tempSourceApp.delete();
-
-    // Return updated config
-    return {
-        ...config,
-        sourceProject,
-        destProject,
-        collections: selectedCollections,
-        includeSubcollections,
-        dryRun,
-        merge,
-    };
-}
-
 // =============================================================================
 // Firebase
 // =============================================================================
@@ -1035,653 +364,46 @@ function initializeFirebase(config: Config): { sourceDb: Firestore; destDb: Fire
     };
 }
 
-async function cleanupFirebase(): Promise<void> {
-    if (sourceApp) await sourceApp.delete();
-    if (destApp) await destApp.delete();
-}
-
-async function getSubcollections(docRef: DocumentReference): Promise<string[]> {
-    const collections = await docRef.listCollections();
-    return collections.map((col) => col.id);
-}
-
-function getDestCollectionPath(
-    sourcePath: string,
-    renameMapping: Record<string, string>
-): string {
-    // Get the root collection name from the source path
-    const rootCollection = sourcePath.split('/')[0];
-
-    // Check if this root collection should be renamed
-    if (renameMapping[rootCollection]) {
-        // Replace the root collection name with the destination name
-        return renameMapping[rootCollection] + sourcePath.slice(rootCollection.length);
-    }
-
-    return sourcePath;
-}
-
-function getDestDocId(
-    sourceId: string,
-    prefix: string | null,
-    suffix: string | null
-): string {
-    let destId = sourceId;
-    if (prefix) {
-        destId = prefix + destId;
-    }
-    if (suffix) {
-        destId = destId + suffix;
-    }
-    return destId;
-}
-
-// =============================================================================
-// Webhook
-// =============================================================================
-
-interface WebhookPayload {
-    source: string;
-    destination: string;
-    collections: string[];
-    stats: Stats;
-    duration: number;
-    dryRun: boolean;
-    success: boolean;
-    error?: string;
-}
-
-function detectWebhookType(url: string): 'slack' | 'discord' | 'custom' {
-    if (url.includes('hooks.slack.com')) {
-        return 'slack';
-    }
-    if (url.includes('discord.com/api/webhooks')) {
-        return 'discord';
-    }
-    return 'custom';
-}
-
-function formatSlackPayload(payload: WebhookPayload): Record<string, unknown> {
-    const status = payload.success ? ':white_check_mark: Success' : ':x: Failed';
-    const mode = payload.dryRun ? ' (DRY RUN)' : '';
-
-    const fields = [
-        { title: 'Source', value: payload.source, short: true },
-        { title: 'Destination', value: payload.destination, short: true },
-        { title: 'Collections', value: payload.collections.join(', '), short: false },
-        { title: 'Transferred', value: String(payload.stats.documentsTransferred), short: true },
-        { title: 'Deleted', value: String(payload.stats.documentsDeleted), short: true },
-        { title: 'Errors', value: String(payload.stats.errors), short: true },
-        { title: 'Duration', value: `${payload.duration}s`, short: true },
-    ];
-
-    if (payload.error) {
-        fields.push({ title: 'Error', value: payload.error, short: false });
-    }
-
-    return {
-        attachments: [
-            {
-                color: payload.success ? '#36a64f' : '#ff0000',
-                title: `fscopy Transfer${mode}`,
-                text: status,
-                fields,
-                footer: 'fscopy',
-                ts: Math.floor(Date.now() / 1000),
-            },
-        ],
-    };
-}
-
-function formatDiscordPayload(payload: WebhookPayload): Record<string, unknown> {
-    const status = payload.success ? '✅ Success' : '❌ Failed';
-    const mode = payload.dryRun ? ' (DRY RUN)' : '';
-    const color = payload.success ? 0x36a64f : 0xff0000;
-
-    const fields = [
-        { name: 'Source', value: payload.source, inline: true },
-        { name: 'Destination', value: payload.destination, inline: true },
-        { name: 'Collections', value: payload.collections.join(', '), inline: false },
-        { name: 'Transferred', value: String(payload.stats.documentsTransferred), inline: true },
-        { name: 'Deleted', value: String(payload.stats.documentsDeleted), inline: true },
-        { name: 'Errors', value: String(payload.stats.errors), inline: true },
-        { name: 'Duration', value: `${payload.duration}s`, inline: true },
-    ];
-
-    if (payload.error) {
-        fields.push({ name: 'Error', value: payload.error, inline: false });
-    }
-
-    return {
-        embeds: [
-            {
-                title: `fscopy Transfer${mode}`,
-                description: status,
-                color,
-                fields,
-                footer: { text: 'fscopy' },
-                timestamp: new Date().toISOString(),
-            },
-        ],
-    };
-}
-
-async function sendWebhook(
-    webhookUrl: string,
-    payload: WebhookPayload,
-    logger: Logger
+async function checkDatabaseConnectivity(
+    sourceDb: Firestore,
+    destDb: Firestore,
+    config: Config
 ): Promise<void> {
-    const webhookType = detectWebhookType(webhookUrl);
-
-    let body: Record<string, unknown>;
-    switch (webhookType) {
-        case 'slack':
-            body = formatSlackPayload(payload);
-            break;
-        case 'discord':
-            body = formatDiscordPayload(payload);
-            break;
-        default:
-            body = payload as unknown as Record<string, unknown>;
-    }
-
-    try {
-        const response = await fetch(webhookUrl, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify(body),
-        });
-
-        if (!response.ok) {
-            const errorText = await response.text();
-            throw new Error(`HTTP ${response.status}: ${errorText}`);
-        }
-
-        logger.info(`Webhook sent successfully (${webhookType})`, { url: webhookUrl });
-        console.log(`📤 Webhook notification sent (${webhookType})`);
-    } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        logger.error(`Failed to send webhook: ${message}`, { url: webhookUrl });
-        console.error(`⚠️  Failed to send webhook: ${message}`);
-    }
-}
-
-// =============================================================================
-// State Management (Resume Support)
-// =============================================================================
+    console.log('🔌 Checking database connectivity...');
 
-const STATE_VERSION = 1;
-
-function loadTransferState(stateFile: string): TransferState | null {
+    // Check source database
     try {
-        if (!fs.existsSync(stateFile)) {
-            return null;
-        }
-        const content = fs.readFileSync(stateFile, 'utf-8');
-        const state = JSON.parse(content) as TransferState;
-
-        if (state.version !== STATE_VERSION) {
-            console.warn(`⚠️  State file version mismatch (expected ${STATE_VERSION}, got ${state.version})`);
-            return null;
-        }
-
-        return state;
+        await sourceDb.listCollections();
+        console.log(`   ✓ Source (${config.sourceProject}) - connected`);
     } catch (error) {
-        console.error(`⚠️  Failed to load state file: ${(error as Error).message}`);
-        return null;
+        const err = error as Error & { code?: string };
+        const errorInfo = formatFirebaseError(err);
+        const hint = errorInfo.suggestion ? `\n   Hint: ${errorInfo.suggestion}` : '';
+        throw new Error(`Cannot connect to source database (${config.sourceProject}): ${errorInfo.message}${hint}`);
     }
-}
 
-function saveTransferState(stateFile: string, state: TransferState): void {
-    state.updatedAt = new Date().toISOString();
-    fs.writeFileSync(stateFile, JSON.stringify(state, null, 2));
-}
-
-function deleteTransferState(stateFile: string): void {
-    try {
-        if (fs.existsSync(stateFile)) {
-            fs.unlinkSync(stateFile);
-        }
-    } catch {
-        // Ignore errors when deleting state file
-    }
-}
-
-function createInitialState(config: Config): TransferState {
-    return {
-        version: STATE_VERSION,
-        sourceProject: config.sourceProject!,
-        destProject: config.destProject!,
-        collections: config.collections,
-        startedAt: new Date().toISOString(),
-        updatedAt: new Date().toISOString(),
-        completedDocs: {},
-        stats: {
-            collectionsProcessed: 0,
-            documentsTransferred: 0,
-            documentsDeleted: 0,
-            errors: 0,
-        },
-    };
-}
-
-function validateStateForResume(state: TransferState, config: Config): string[] {
-    const errors: string[] = [];
-
-    if (state.sourceProject !== config.sourceProject) {
-        errors.push(`Source project mismatch: state has "${state.sourceProject}", config has "${config.sourceProject}"`);
-    }
-    if (state.destProject !== config.destProject) {
-        errors.push(`Destination project mismatch: state has "${state.destProject}", config has "${config.destProject}"`);
-    }
-
-    // Check if collections are compatible (state collections should be subset of config)
-    const configCollections = new Set(config.collections);
-    for (const col of state.collections) {
-        if (!configCollections.has(col)) {
-            errors.push(`State contains collection "${col}" not in current config`);
-        }
-    }
-
-    return errors;
-}
-
-function isDocCompleted(state: TransferState, collectionPath: string, docId: string): boolean {
-    const completedInCollection = state.completedDocs[collectionPath];
-    return completedInCollection ? completedInCollection.includes(docId) : false;
-}
-
-function markDocCompleted(state: TransferState, collectionPath: string, docId: string): void {
-    if (!state.completedDocs[collectionPath]) {
-        state.completedDocs[collectionPath] = [];
-    }
-    state.completedDocs[collectionPath].push(docId);
-}
-
-// =============================================================================
-// Transfer Logic
-// =============================================================================
-
-async function clearCollection(
-    db: Firestore,
-    collectionPath: string,
-    config: Config,
-    logger: Logger,
-    includeSubcollections: boolean
-): Promise<number> {
-    let deletedCount = 0;
-    const collectionRef = db.collection(collectionPath);
-    const snapshot = await collectionRef.get();
-
-    if (snapshot.empty) {
-        return 0;
-    }
-
-    // Delete subcollections first if enabled
-    if (includeSubcollections) {
-        for (const doc of snapshot.docs) {
-            const subcollections = await getSubcollections(doc.ref);
-            for (const subId of subcollections) {
-                // Check exclude patterns
-                if (matchesExcludePattern(subId, config.exclude)) {
-                    continue;
-                }
-                const subPath = `${collectionPath}/${doc.id}/${subId}`;
-                deletedCount += await clearCollection(db, subPath, config, logger, true);
-            }
-        }
-    }
-
-    // Delete documents in batches
-    const docs = snapshot.docs;
-    for (let i = 0; i < docs.length; i += config.batchSize) {
-        const batch = docs.slice(i, i + config.batchSize);
-        const writeBatch = db.batch();
-
-        for (const doc of batch) {
-            writeBatch.delete(doc.ref);
-            deletedCount++;
-        }
-
-        if (!config.dryRun) {
-            await withRetry(() => writeBatch.commit(), {
-                retries: config.retries,
-                onRetry: (attempt, max, err, delay) => {
-                    logger.error(`Retry delete ${attempt}/${max} for ${collectionPath}`, {
-                        error: err.message,
-                        delay,
-                    });
-                },
-            });
-        }
-
-        logger.info(`Deleted ${batch.length} documents from ${collectionPath}`);
-    }
-
-    return deletedCount;
-}
-
-async function deleteOrphanDocuments(
-    sourceDb: Firestore,
-    destDb: Firestore,
-    sourceCollectionPath: string,
-    config: Config,
-    logger: Logger
-): Promise<number> {
-    let deletedCount = 0;
-
-    // Get the destination path (may be renamed)
-    const destCollectionPath = getDestCollectionPath(sourceCollectionPath, config.renameCollection);
-
-    // Get all document IDs from source
-    const sourceSnapshot = await sourceDb.collection(sourceCollectionPath).get();
-    const sourceIds = new Set(sourceSnapshot.docs.map((doc) => doc.id));
-
-    // Get all document IDs from destination
-    const destSnapshot = await destDb.collection(destCollectionPath).get();
-
-    // Find orphan documents (in dest but not in source)
-    const orphanDocs = destSnapshot.docs.filter((doc) => !sourceIds.has(doc.id));
-
-    if (orphanDocs.length === 0) {
-        return 0;
-    }
-
-    logger.info(`Found ${orphanDocs.length} orphan documents in ${destCollectionPath}`);
-
-    // Delete orphan documents in batches
-    for (let i = 0; i < orphanDocs.length; i += config.batchSize) {
-        const batch = orphanDocs.slice(i, i + config.batchSize);
-        const writeBatch = destDb.batch();
-
-        for (const doc of batch) {
-            // If subcollections are included, recursively delete orphans in subcollections first
-            if (config.includeSubcollections) {
-                const subcollections = await getSubcollections(doc.ref);
-                for (const subId of subcollections) {
-                    if (matchesExcludePattern(subId, config.exclude)) {
-                        continue;
-                    }
-                    const subPath = `${destCollectionPath}/${doc.id}/${subId}`;
-                    // For orphan parent docs, clear all subcollection data
-                    deletedCount += await clearCollection(destDb, subPath, config, logger, true);
-                }
-            }
-
-            writeBatch.delete(doc.ref);
-            deletedCount++;
-        }
-
-        if (!config.dryRun) {
-            await withRetry(() => writeBatch.commit(), {
-                retries: config.retries,
-                onRetry: (attempt, max, err, delay) => {
-                    logger.error(`Retry delete orphans ${attempt}/${max} for ${destCollectionPath}`, {
-                        error: err.message,
-                        delay,
-                    });
-                },
-            });
-        }
-
-        logger.info(`Deleted ${batch.length} orphan documents from ${destCollectionPath}`);
-    }
-
-    // Also check subcollections of existing documents for orphans
-    if (config.includeSubcollections) {
-        for (const sourceDoc of sourceSnapshot.docs) {
-            const sourceSubcollections = await getSubcollections(sourceDoc.ref);
-            for (const subId of sourceSubcollections) {
-                if (matchesExcludePattern(subId, config.exclude)) {
-                    continue;
-                }
-                const subPath = `${sourceCollectionPath}/${sourceDoc.id}/${subId}`;
-                deletedCount += await deleteOrphanDocuments(sourceDb, destDb, subPath, config, logger);
-            }
-        }
-    }
-
-    return deletedCount;
-}
-
-async function countDocuments(
-    sourceDb: Firestore,
-    collectionPath: string,
-    config: Config,
-    depth: number = 0
-): Promise<number> {
-    let count = 0;
-
-    // Build query with where filters (only at root level)
-    let query: FirebaseFirestore.Query = sourceDb.collection(collectionPath);
-    if (depth === 0 && config.where.length > 0) {
-        for (const filter of config.where) {
-            query = query.where(filter.field, filter.operator, filter.value);
-        }
-    }
-
-    const snapshot = await query.get();
-    count += snapshot.size;
-
-    if (config.includeSubcollections) {
-        for (const doc of snapshot.docs) {
-            const subcollections = await getSubcollections(doc.ref);
-            for (const subId of subcollections) {
-                const subPath = `${collectionPath}/${doc.id}/${subId}`;
-
-                // Check exclude patterns
-                if (matchesExcludePattern(subId, config.exclude)) {
-                    continue;
-                }
-
-                count += await countDocuments(sourceDb, subPath, config, depth + 1);
-            }
-        }
-    }
-
-    return count;
-}
-
-interface TransferContext {
-    sourceDb: Firestore;
-    destDb: Firestore;
-    config: Config;
-    stats: Stats;
-    logger: Logger;
-    progressBar: cliProgress.SingleBar | null;
-    transformFn: TransformFunction | null;
-    state: TransferState | null;
-}
-
-async function transferCollection(
-    ctx: TransferContext,
-    collectionPath: string,
-    depth: number = 0
-): Promise<void> {
-    const { sourceDb, destDb, config, stats, logger, progressBar, transformFn, state } = ctx;
-
-    // Get the destination path (may be renamed)
-    const destCollectionPath = getDestCollectionPath(collectionPath, config.renameCollection);
-
-    const sourceCollectionRef = sourceDb.collection(collectionPath);
-    let query: FirebaseFirestore.Query = sourceCollectionRef;
-
-    // Apply where filters (only at root level)
-    if (depth === 0 && config.where.length > 0) {
-        for (const filter of config.where) {
-            query = query.where(filter.field, filter.operator, filter.value);
+    // Check destination database (only if different from source)
+    if (config.sourceProject !== config.destProject) {
+        try {
+            await destDb.listCollections();
+            console.log(`   ✓ Destination (${config.destProject}) - connected`);
+        } catch (error) {
+            const err = error as Error & { code?: string };
+            const errorInfo = formatFirebaseError(err);
+            const hint = errorInfo.suggestion ? `\n   Hint: ${errorInfo.suggestion}` : '';
+            throw new Error(`Cannot connect to destination database (${config.destProject}): ${errorInfo.message}${hint}`);
         }
+    } else {
+        console.log(`   ✓ Destination (same as source) - connected`);
     }
 
-    if (config.limit > 0 && depth === 0) {
-        query = query.limit(config.limit);
-    }
-
-    const snapshot = await withRetry(() => query.get(), {
-        retries: config.retries,
-        onRetry: (attempt, max, err, delay) => {
-            logger.error(`Retry ${attempt}/${max} for ${collectionPath}`, {
-                error: err.message,
-                delay,
-            });
-        },
-    });
-
-    if (snapshot.empty) {
-        return;
-    }
-
-    stats.collectionsProcessed++;
-    logger.info(`Processing collection: ${collectionPath}`, { documents: snapshot.size });
-
-    const docs = snapshot.docs;
-    const batchDocIds: string[] = []; // Track docs in current batch for state saving
-
-    for (let i = 0; i < docs.length; i += config.batchSize) {
-        const batch = docs.slice(i, i + config.batchSize);
-        const destBatch: WriteBatch = destDb.batch();
-        batchDocIds.length = 0; // Clear for new batch
-
-        for (const doc of batch) {
-            // Skip if already completed (resume mode)
-            if (state && isDocCompleted(state, collectionPath, doc.id)) {
-                if (progressBar) {
-                    progressBar.increment();
-                }
-                stats.documentsTransferred++;
-                continue;
-            }
-
-            // Get destination document ID (with optional prefix/suffix)
-            const destDocId = getDestDocId(doc.id, config.idPrefix, config.idSuffix);
-            const destDocRef = destDb.collection(destCollectionPath).doc(destDocId);
-
-            // Apply transform if provided
-            let docData = doc.data() as Record<string, unknown>;
-            if (transformFn) {
-                const transformed = transformFn(docData, {
-                    id: doc.id,
-                    path: `${collectionPath}/${doc.id}`,
-                });
-                if (transformed === null) {
-                    // Skip this document if transform returns null
-                    logger.info('Skipped document (transform returned null)', {
-                        collection: collectionPath,
-                        docId: doc.id,
-                    });
-                    if (progressBar) {
-                        progressBar.increment();
-                    }
-                    // Mark as completed even if skipped
-                    batchDocIds.push(doc.id);
-                    continue;
-                }
-                docData = transformed;
-            }
-
-            if (!config.dryRun) {
-                // Use merge option if enabled
-                if (config.merge) {
-                    destBatch.set(destDocRef, docData, { merge: true });
-                } else {
-                    destBatch.set(destDocRef, docData);
-                }
-            }
-
-            batchDocIds.push(doc.id);
-            stats.documentsTransferred++;
-            if (progressBar) {
-                progressBar.increment();
-            }
-
-            logger.info('Transferred document', {
-                source: collectionPath,
-                dest: destCollectionPath,
-                sourceDocId: doc.id,
-                destDocId: destDocId,
-            });
-
-            if (config.includeSubcollections) {
-                const subcollections = await getSubcollections(doc.ref);
-
-                for (const subcollectionId of subcollections) {
-                    // Check exclude patterns
-                    if (matchesExcludePattern(subcollectionId, config.exclude)) {
-                        logger.info(`Skipping excluded subcollection: ${subcollectionId}`);
-                        continue;
-                    }
-
-                    const subcollectionPath = `${collectionPath}/${doc.id}/${subcollectionId}`;
-
-                    await transferCollection(
-                        { ...ctx, config: { ...config, limit: 0, where: [] } },
-                        subcollectionPath,
-                        depth + 1
-                    );
-                }
-            }
-        }
-
-        if (!config.dryRun && batch.length > 0) {
-            await withRetry(() => destBatch.commit(), {
-                retries: config.retries,
-                onRetry: (attempt, max, err, delay) => {
-                    logger.error(`Retry commit ${attempt}/${max}`, { error: err.message, delay });
-                },
-            });
-
-            // Save state after successful batch commit (for resume support)
-            if (state && batchDocIds.length > 0) {
-                for (const docId of batchDocIds) {
-                    markDocCompleted(state, collectionPath, docId);
-                }
-                state.stats = { ...stats };
-                saveTransferState(config.stateFile, state);
-            }
-        }
-    }
+    console.log('');
 }
 
-// =============================================================================
-// Parallel Processing Helper
-// =============================================================================
-
-async function processInParallel<T, R>(
-    items: T[],
-    concurrency: number,
-    processor: (item: T) => Promise<R>
-): Promise<R[]> {
-    const results: R[] = [];
-    const executing: Promise<void>[] = [];
-
-    for (const item of items) {
-        const promise = processor(item).then((result) => {
-            results.push(result);
-        });
-
-        executing.push(promise);
-
-        if (executing.length >= concurrency) {
-            await Promise.race(executing);
-            // Remove completed promises
-            for (let i = executing.length - 1; i >= 0; i--) {
-                const p = executing[i];
-                // Check if promise is settled by racing with resolved promise
-                const isSettled = await Promise.race([
-                    p.then(() => true).catch(() => true),
-                    Promise.resolve(false),
-                ]);
-                if (isSettled) {
-                    executing.splice(i, 1);
-                }
-            }
-        }
-    }
-
-    await Promise.all(executing);
-    return results;
+async function cleanupFirebase(): Promise<void> {
+    if (sourceApp) await sourceApp.delete();
+    if (destApp) await destApp.delete();
 }
-
 // =============================================================================
 // Main
 // =============================================================================
@@ -1693,6 +415,9 @@ if (argv.init !== undefined) {
     process.exit(0);
 }
 
+// Check credentials before proceeding
+ensureCredentials();
+
 // Main transfer flow
 let config: Config = defaults;
 let logger: Logger | null = null;
@@ -1773,6 +498,45 @@ try {
 
     const { sourceDb, destDb } = initializeFirebase(config);
 
+    // Verify database connectivity before proceeding
+    await checkDatabaseConnectivity(sourceDb, destDb, config);
+
+    // Validate transform with sample documents (in dry-run mode)
+    if (transformFn && config.dryRun) {
+        console.log('🧪 Validating transform with sample documents...');
+        let samplesTested = 0;
+        let samplesSkipped = 0;
+        let samplesErrors = 0;
+
+        for (const collection of config.collections) {
+            const snapshot = await sourceDb.collection(collection).limit(3).get();
+            for (const doc of snapshot.docs) {
+                try {
+                    const result = transformFn(doc.data() as Record<string, unknown>, {
+                        id: doc.id,
+                        path: `${collection}/${doc.id}`,
+                    });
+                    if (result === null) {
+                        samplesSkipped++;
+                    } else {
+                        samplesTested++;
+                    }
+                } catch (error) {
+                    samplesErrors++;
+                    const err = error as Error;
+                    console.error(`   ⚠️  Transform error on ${collection}/${doc.id}: ${err.message}`);
+                }
+            }
+        }
+
+        if (samplesErrors > 0) {
+            console.log(`   ❌ ${samplesErrors} sample(s) failed - review your transform function`);
+        } else if (samplesTested > 0 || samplesSkipped > 0) {
+            console.log(`   ✓ Tested ${samplesTested} sample(s), ${samplesSkipped} would be skipped`);
+        }
+        console.log('');
+    }
+
     if (!config.resume) {
         stats = {
             collectionsProcessed: 0,
@@ -1788,19 +552,66 @@ try {
 
     if (!argv.quiet) {
         console.log('📊 Counting documents...');
+        let lastSubcollectionLog = Date.now();
+        let subcollectionCount = 0;
+
+        const countProgress: CountProgress = {
+            onCollection: (path, count) => {
+                console.log(`   ${path}: ${count} documents`);
+            },
+            onSubcollection: (_path) => {
+                subcollectionCount++;
+                // Show progress every 2 seconds to avoid flooding the console
+                const now = Date.now();
+                if (now - lastSubcollectionLog > 2000) {
+                    process.stdout.write(`\r   Scanning subcollections... (${subcollectionCount} found)`);
+                    lastSubcollectionLog = now;
+                }
+            },
+        };
+
         for (const collection of config.collections) {
-            totalDocs += await countDocuments(sourceDb, collection, config);
+            totalDocs += await countDocuments(sourceDb, collection, config, 0, countProgress);
+        }
+
+        // Clear the subcollection line if any were found
+        if (subcollectionCount > 0) {
+            process.stdout.write('\r' + ' '.repeat(60) + '\r');
+            console.log(`   Subcollections scanned: ${subcollectionCount}`);
         }
-        console.log(`   Found ${totalDocs} documents to transfer\n`);
+        console.log(`   Total: ${totalDocs} documents to transfer\n`);
 
         if (totalDocs > 0) {
             progressBar = new cliProgress.SingleBar({
-                format: '📦 Progress |{bar}| {percentage}% | {value}/{total} docs | ETA: {eta}s',
+                format: '📦 Progress |{bar}| {percentage}% | {value}/{total} docs | {speed} docs/s | ETA: {eta}s',
                 barCompleteChar: '█',
                 barIncompleteChar: '░',
                 hideCursor: true,
             });
-            progressBar.start(totalDocs, 0);
+            progressBar.start(totalDocs, 0, { speed: '0' });
+
+            // Track speed using transfer stats
+            let lastDocsTransferred = 0;
+            let lastTime = Date.now();
+
+            const speedInterval = setInterval(() => {
+                if (progressBar) {
+                    const now = Date.now();
+                    const timeDiff = (now - lastTime) / 1000;
+                    const currentDocs = stats.documentsTransferred;
+
+                    if (timeDiff > 0) {
+                        const docsDiff = currentDocs - lastDocsTransferred;
+                        const speed = Math.round(docsDiff / timeDiff);
+                        lastDocsTransferred = currentDocs;
+                        lastTime = now;
+                        progressBar.update({ speed: String(speed) });
+                    }
+                }
+            }, 500);
+
+            // Store interval for cleanup
+            (progressBar as unknown as { _speedInterval: NodeJS.Timeout })._speedInterval = speedInterval;
         }
     }
 
@@ -1821,20 +632,43 @@ try {
         console.log(`   Deleted ${stats.documentsDeleted} documents\n`);
     }
 
-    const ctx: TransferContext = { sourceDb, destDb, config, stats, logger, progressBar, transformFn, state: transferState };
+    // Create rate limiter if enabled
+    const rateLimiter = config.rateLimit > 0 ? new RateLimiter(config.rateLimit) : null;
+    if (rateLimiter) {
+        console.log(`⏱️  Rate limiting enabled: ${config.rateLimit} docs/s\n`);
+    }
+
+    const ctx: TransferContext = { sourceDb, destDb, config, stats, logger, progressBar, transformFn, state: transferState, rateLimiter };
 
     // Transfer collections (with optional parallelism)
     if (config.parallel > 1) {
-        await processInParallel(config.collections, config.parallel, (collection) =>
+        const { errors } = await processInParallel(config.collections, config.parallel, (collection) =>
             transferCollection(ctx, collection)
         );
+        if (errors.length > 0) {
+            for (const err of errors) {
+                logger.error('Parallel transfer error', { error: err.message });
+                stats.errors++;
+            }
+        }
     } else {
         for (const collection of config.collections) {
-            await transferCollection(ctx, collection);
+            try {
+                await transferCollection(ctx, collection);
+            } catch (error) {
+                const err = error instanceof Error ? error : new Error(String(error));
+                logger.error(`Transfer failed for ${collection}`, { error: err.message });
+                stats.errors++;
+            }
         }
     }
 
     if (progressBar) {
+        // Clear speed update interval
+        const interval = (progressBar as unknown as { _speedInterval?: NodeJS.Timeout })._speedInterval;
+        if (interval) {
+            clearInterval(interval);
+        }
         progressBar.stop();
     }
 
@@ -1866,30 +700,95 @@ try {
     });
     logger.summary(stats, duration);
 
-    console.log('\n' + '='.repeat(60));
-    console.log('📊 TRANSFER SUMMARY');
-    console.log('='.repeat(60));
-    console.log(`Collections processed: ${stats.collectionsProcessed}`);
-    if (stats.documentsDeleted > 0) {
-        console.log(`Documents deleted:     ${stats.documentsDeleted}`);
+    // Verify transfer if enabled (and not dry-run)
+    let verifyResult: Record<string, { source: number; dest: number; match: boolean }> | null = null;
+    if (config.verify && !config.dryRun) {
+        if (!config.json) {
+            console.log('\n🔍 Verifying transfer...');
+        }
+        verifyResult = {};
+        let verifyPassed = true;
+
+        for (const collection of config.collections) {
+            const destCollection = getDestCollectionPath(collection, config.renameCollection);
+
+            // Count source documents
+            const sourceCount = await sourceDb.collection(collection).count().get();
+            const sourceTotal = sourceCount.data().count;
+
+            // Count destination documents
+            const destCount = await destDb.collection(destCollection).count().get();
+            const destTotal = destCount.data().count;
+
+            const match = sourceTotal === destTotal;
+            verifyResult[collection] = { source: sourceTotal, dest: destTotal, match };
+
+            if (!config.json) {
+                if (match) {
+                    console.log(`   ✓ ${collection}: ${sourceTotal} docs (matched)`);
+                } else {
+                    console.log(`   ⚠️  ${collection}: source=${sourceTotal}, dest=${destTotal} (mismatch)`);
+                }
+            }
+            if (!match) verifyPassed = false;
+        }
+
+        if (!config.json) {
+            if (verifyPassed) {
+                console.log('   ✓ Verification passed');
+            } else {
+                console.log('   ⚠️  Verification found mismatches');
+            }
+        }
     }
-    console.log(`Documents transferred: ${stats.documentsTransferred}`);
-    console.log(`Errors: ${stats.errors}`);
-    console.log(`Duration: ${duration}s`);
 
-    if (argv.log) {
-        console.log(`Log file: ${argv.log}`);
+    // Delete state file on successful completion (before JSON output)
+    if (!config.dryRun) {
+        deleteTransferState(config.stateFile);
     }
 
-    if (config.dryRun) {
-        console.log('\n⚠ DRY RUN: No data was actually written');
-        console.log('   Run with --dry-run=false to perform the transfer');
+    // JSON output mode
+    if (config.json) {
+        const jsonOutput = {
+            success: true,
+            dryRun: config.dryRun,
+            source: config.sourceProject,
+            destination: config.destProject,
+            collections: config.collections,
+            stats: {
+                collectionsProcessed: stats.collectionsProcessed,
+                documentsTransferred: stats.documentsTransferred,
+                documentsDeleted: stats.documentsDeleted,
+                errors: stats.errors,
+            },
+            duration: Number.parseFloat(duration),
+            verify: verifyResult,
+        };
+        console.log(JSON.stringify(jsonOutput, null, 2));
     } else {
-        console.log('\n✓ Transfer completed successfully');
-        // Delete state file on successful completion
-        deleteTransferState(config.stateFile);
+        console.log('\n' + '='.repeat(60));
+        console.log('📊 TRANSFER SUMMARY');
+        console.log('='.repeat(60));
+        console.log(`Collections processed: ${stats.collectionsProcessed}`);
+        if (stats.documentsDeleted > 0) {
+            console.log(`Documents deleted:     ${stats.documentsDeleted}`);
+        }
+        console.log(`Documents transferred: ${stats.documentsTransferred}`);
+        console.log(`Errors: ${stats.errors}`);
+        console.log(`Duration: ${duration}s`);
+
+        if (argv.log) {
+            console.log(`Log file: ${argv.log}`);
+        }
+
+        if (config.dryRun) {
+            console.log('\n⚠ DRY RUN: No data was actually written');
+            console.log('   Run with --dry-run=false to perform the transfer');
+        } else {
+            console.log('\n✓ Transfer completed successfully');
+        }
+        console.log('='.repeat(60) + '\n');
     }
-    console.log('='.repeat(60) + '\n');
 
     // Send webhook notification if configured
     if (config.webhook) {
@@ -1911,7 +810,29 @@ try {
     await cleanupFirebase();
 } catch (error) {
     const errorMessage = (error as Error).message;
-    console.error('\n❌ Error during transfer:', errorMessage);
+    const duration = Number.parseFloat(((Date.now() - startTime) / 1000).toFixed(2));
+
+    // JSON output mode for errors
+    if (config.json) {
+        const jsonOutput = {
+            success: false,
+            error: errorMessage,
+            dryRun: config.dryRun,
+            source: config.sourceProject,
+            destination: config.destProject,
+            collections: config.collections,
+            stats: {
+                collectionsProcessed: stats.collectionsProcessed,
+                documentsTransferred: stats.documentsTransferred,
+                documentsDeleted: stats.documentsDeleted,
+                errors: stats.errors,
+            },
+            duration,
+        };
+        console.log(JSON.stringify(jsonOutput, null, 2));
+    } else {
+        console.error('\n❌ Error during transfer:', errorMessage);
+    }
 
     // Send webhook notification on error if configured
     if (config.webhook && logger) {
@@ -1922,7 +843,7 @@ try {
                 destination: config.destProject ?? 'unknown',
                 collections: config.collections,
                 stats,
-                duration: Number.parseFloat(((Date.now() - startTime) / 1000).toFixed(2)),
+                duration,
                 dryRun: config.dryRun,
                 success: false,
                 error: errorMessage,