@fazetitans/fscopy 1.1.0 → 1.1.2

package/README.md

@@ -33,10 +33,14 @@ Transfer documents between Firebase projects with support for subcollections, fi
 - **Webhook notifications** - Send Slack, Discord, or custom webhooks on completion
 - **Resume transfers** - Continue interrupted transfers from saved state
 - **Interactive mode** - Guided setup with prompts for project and collection selection
-- **Progress bar** - Real-time progress with ETA
+- **Progress bar** - Real-time progress with speed (docs/s) and ETA
 - **Automatic retry** - Exponential backoff on network errors
 - **Dry run mode** - Preview changes before applying (enabled by default)
 - **Flexible config** - INI, JSON, or CLI arguments
+- **Rate limiting** - Control transfer speed to avoid quota issues
+- **Size validation** - Skip oversized documents (>1MB)
+- **JSON output** - Machine-readable output for CI/CD pipelines
+- **Post-transfer verification** - Verify document counts after transfer
 
 ## Installation
 
@@ -158,6 +162,18 @@ fscopy -f config.ini --webhook https://hooks.slack.com/services/...
 
 # Resume an interrupted transfer
 fscopy -f config.ini --resume
+
+# Verify document counts after transfer
+fscopy -f config.ini --verify
+
+# Rate limit to 100 docs/second (avoid quota issues)
+fscopy -f config.ini --rate-limit 100
+
+# Skip documents larger than 1MB
+fscopy -f config.ini --skip-oversized
+
+# JSON output for CI/CD pipelines
+fscopy -f config.ini --json
 ```
 
 ### Collection Renaming
@@ -364,6 +380,10 @@ fscopy --init config.json
 | `--webhook`                |       | string  |         | Webhook URL for notifications     |
 | `--resume`                 |       | boolean | `false` | Resume from saved state           |
 | `--state-file`             |       | string  | `.fscopy-state.json` | State file path      |
+| `--verify`                 |       | boolean | `false` | Verify counts after transfer      |
+| `--rate-limit`             |       | number  | `0`     | Limit docs/second (0 = unlimited) |
+| `--skip-oversized`         |       | boolean | `false` | Skip documents > 1MB              |
+| `--json`                   |       | boolean | `false` | JSON output for CI/CD             |
 
 ## How It Works
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@fazetitans/fscopy",
-    "version": "1.1.0",
+    "version": "1.1.2",
     "description": "Fast CLI tool to copy Firestore collections between Firebase projects with filtering, parallel transfers, and subcollection support",
     "type": "module",
     "bin": {
@@ -46,7 +46,8 @@
         "node": ">=18.0.0"
     },
     "files": [
-        "src/cli.ts",
+        "src/**/*.ts",
+        "!src/__tests__/**",
         "README.md",
         "LICENSE"
     ],

package/src/config/defaults.ts

@@ -0,0 +1,88 @@
+import type { Config } from '../types.js';
+
+export const defaults: Config = {
+    collections: [],
+    includeSubcollections: false,
+    dryRun: true,
+    batchSize: 500,
+    limit: 0,
+    sourceProject: null,
+    destProject: null,
+    retries: 3,
+    where: [],
+    exclude: [],
+    merge: false,
+    parallel: 1,
+    clear: false,
+    deleteMissing: false,
+    transform: null,
+    renameCollection: {},
+    idPrefix: null,
+    idSuffix: null,
+    webhook: null,
+    resume: false,
+    stateFile: '.fscopy-state.json',
+    verify: false,
+    rateLimit: 0,
+    skipOversized: false,
+    json: false,
+};
+
+export const iniTemplate = `; fscopy configuration file
+
+[projects]
+source = my-source-project
+dest = my-dest-project
+
+[transfer]
+; Comma-separated list of collections
+collections = collection1, collection2
+includeSubcollections = false
+dryRun = true
+batchSize = 500
+limit = 0
+
+[options]
+; Filter documents: "field operator value" (operators: ==, !=, <, >, <=, >=)
+; where = status == active
+; Exclude subcollections by pattern (comma-separated, supports glob)
+; exclude = logs, temp/*, cache
+; Merge documents instead of overwriting
+merge = false
+; Number of parallel collection transfers
+parallel = 1
+; Clear destination collections before transfer (DESTRUCTIVE)
+clear = false
+; Delete destination docs not present in source (sync mode)
+deleteMissing = false
+; Transform documents during transfer (path to JS/TS file)
+; transform = ./transforms/anonymize.ts
+; Rename collections in destination (format: source:dest, comma-separated)
+; renameCollection = users:users_backup, orders:orders_2024
+; Add prefix or suffix to document IDs
+; idPrefix = backup_
+; idSuffix = _v2
+; Webhook URL for transfer notifications (Slack, Discord, or custom)
+; webhook = https://hooks.slack.com/services/...
+`;
+
+export const jsonTemplate = {
+    sourceProject: 'my-source-project',
+    destProject: 'my-dest-project',
+    collections: ['collection1', 'collection2'],
+    includeSubcollections: false,
+    dryRun: true,
+    batchSize: 500,
+    limit: 0,
+    where: [],
+    exclude: [],
+    merge: false,
+    parallel: 1,
+    clear: false,
+    deleteMissing: false,
+    transform: null,
+    renameCollection: {},
+    idPrefix: null,
+    idSuffix: null,
+    webhook: null,
+};

package/src/config/generator.ts

@@ -0,0 +1,27 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { getFileFormat } from './parser.js';
+import { iniTemplate, jsonTemplate } from './defaults.js';
+
+export function generateConfigFile(outputPath: string): boolean {
+    const filePath = path.resolve(outputPath);
+    const format = getFileFormat(filePath);
+
+    if (fs.existsSync(filePath)) {
+        console.error(`❌ File already exists: ${filePath}`);
+        console.error('   Use a different filename or delete the existing file.');
+        process.exitCode = 1;
+        return false;
+    }
+
+    const content = format === 'json' ? JSON.stringify(jsonTemplate, null, 4) : iniTemplate;
+
+    fs.writeFileSync(filePath, content, 'utf-8');
+
+    console.log(`✓ Config template created: ${filePath}`);
+    console.log('');
+    console.log('Edit the file to configure your transfer, then run:');
+    console.log(`  fscopy -f ${outputPath}`);
+
+    return true;
+}

package/src/config/index.ts

@@ -0,0 +1,15 @@
+export { defaults, iniTemplate, jsonTemplate } from './defaults.js';
+export {
+    getFileFormat,
+    parseBoolean,
+    parseWhereFilter,
+    parseWhereFilters,
+    parseStringList,
+    parseRenameMapping,
+    parseIniConfig,
+    parseJsonConfig,
+    loadConfigFile,
+    mergeConfig,
+} from './parser.js';
+export { validateConfig } from './validator.js';
+export { generateConfigFile } from './generator.js';

package/src/config/parser.ts

@@ -0,0 +1,261 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import ini from 'ini';
+import type { Config, WhereFilter, CliArgs } from '../types.js';
+
+export function getFileFormat(filePath: string): 'json' | 'ini' {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext === '.json') return 'json';
+    return 'ini';
+}
+
+export function parseBoolean(val: unknown): boolean {
+    if (typeof val === 'boolean') return val;
+    if (typeof val === 'string') {
+        return val.toLowerCase() === 'true';
+    }
+    return false;
+}
+
+export function parseWhereFilter(filterStr: string): WhereFilter | null {
+    const operatorRegex = /(==|!=|<=|>=|<|>)/;
+    const match = new RegExp(operatorRegex).exec(filterStr);
+
+    if (!match) {
+        console.warn(`⚠️  Invalid where filter: "${filterStr}" (missing operator)`);
+        return null;
+    }
+
+    const operator = match[0] as FirebaseFirestore.WhereFilterOp;
+    const [fieldPart, valuePart] = filterStr.split(operatorRegex).filter((_, i) => i !== 1);
+
+    if (!fieldPart || !valuePart) {
+        console.warn(`⚠️  Invalid where filter: "${filterStr}" (missing field or value)`);
+        return null;
+    }
+
+    const field = fieldPart.trim();
+    const rawValue = valuePart.trim();
+
+    let value: string | number | boolean;
+    if (rawValue === 'true') {
+        value = true;
+    } else if (rawValue === 'false') {
+        value = false;
+    } else if (rawValue === 'null') {
+        value = null as unknown as string;
+    } else if (!Number.isNaN(Number(rawValue)) && rawValue !== '') {
+        value = Number(rawValue);
+    } else {
+        value = rawValue.replaceAll(/(?:^["'])|(?:["']$)/g, '');
+    }
+
+    return { field, operator, value };
+}
+
+export function parseWhereFilters(filters: string[] | undefined): WhereFilter[] {
+    if (!filters || filters.length === 0) return [];
+    return filters.map(parseWhereFilter).filter((f): f is WhereFilter => f !== null);
+}
+
+export function parseStringList(value: string | undefined): string[] {
+    if (!value) return [];
+    return value
+        .split(',')
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+
+export function parseRenameMapping(
+    mappings: string[] | string | undefined
+): Record<string, string> {
+    if (!mappings) return {};
+
+    const result: Record<string, string> = {};
+    const items = Array.isArray(mappings) ? mappings : parseStringList(mappings);
+
+    for (const item of items) {
+        const mapping = String(item).trim();
+        const colonIndex = mapping.indexOf(':');
+        if (colonIndex === -1) {
+            console.warn(`⚠️  Invalid rename mapping: "${mapping}" (missing ':')`);
+            continue;
+        }
+        const source = mapping.slice(0, colonIndex).trim();
+        const dest = mapping.slice(colonIndex + 1).trim();
+        if (!source || !dest) {
+            console.warn(`⚠️  Invalid rename mapping: "${mapping}" (empty source or dest)`);
+            continue;
+        }
+        result[source] = dest;
+    }
+
+    return result;
+}
+
+export function parseIniConfig(content: string): Partial<Config> {
+    const parsed = ini.parse(content) as {
+        projects?: { source?: string; dest?: string };
+        transfer?: {
+            collections?: string;
+            includeSubcollections?: string | boolean;
+            dryRun?: string | boolean;
+            batchSize?: string;
+            limit?: string;
+        };
+        options?: {
+            where?: string;
+            exclude?: string;
+            merge?: string | boolean;
+            parallel?: string;
+            clear?: string | boolean;
+            deleteMissing?: string | boolean;
+            transform?: string;
+            renameCollection?: string;
+            idPrefix?: string;
+            idSuffix?: string;
+            webhook?: string;
+        };
+    };
+
+    let collections: string[] = [];
+    if (parsed.transfer?.collections) {
+        collections = parsed.transfer.collections
+            .split(',')
+            .map((c) => c.trim())
+            .filter((c) => c.length > 0);
+    }
+
+    const whereFilters = parsed.options?.where
+        ? parseWhereFilters(parseStringList(parsed.options.where))
+        : [];
+
+    return {
+        sourceProject: parsed.projects?.source ?? null,
+        destProject: parsed.projects?.dest ?? null,
+        collections,
+        includeSubcollections: parseBoolean(parsed.transfer?.includeSubcollections),
+        dryRun: parseBoolean(parsed.transfer?.dryRun ?? 'true'),
+        batchSize: Number.parseInt(parsed.transfer?.batchSize ?? '', 10) || 500,
+        limit: Number.parseInt(parsed.transfer?.limit ?? '', 10) || 0,
+        where: whereFilters,
+        exclude: parseStringList(parsed.options?.exclude),
+        merge: parseBoolean(parsed.options?.merge),
+        parallel: Number.parseInt(parsed.options?.parallel ?? '', 10) || 1,
+        clear: parseBoolean(parsed.options?.clear),
+        deleteMissing: parseBoolean(parsed.options?.deleteMissing),
+        transform: parsed.options?.transform ?? null,
+        renameCollection: parseRenameMapping(parsed.options?.renameCollection),
+        idPrefix: parsed.options?.idPrefix ?? null,
+        idSuffix: parsed.options?.idSuffix ?? null,
+        webhook: parsed.options?.webhook ?? null,
+    };
+}
+
+export function parseJsonConfig(content: string): Partial<Config> {
+    const config = JSON.parse(content) as {
+        sourceProject?: string;
+        destProject?: string;
+        collections?: string[];
+        includeSubcollections?: boolean;
+        dryRun?: boolean;
+        batchSize?: number;
+        limit?: number;
+        where?: string[];
+        exclude?: string[];
+        merge?: boolean;
+        parallel?: number;
+        clear?: boolean;
+        deleteMissing?: boolean;
+        transform?: string;
+        renameCollection?: Record<string, string>;
+        idPrefix?: string;
+        idSuffix?: string;
+        webhook?: string;
+    };
+
+    return {
+        sourceProject: config.sourceProject ?? null,
+        destProject: config.destProject ?? null,
+        collections: config.collections,
+        includeSubcollections: config.includeSubcollections,
+        dryRun: config.dryRun,
+        batchSize: config.batchSize,
+        limit: config.limit,
+        where: parseWhereFilters(config.where),
+        exclude: config.exclude,
+        merge: config.merge,
+        parallel: config.parallel,
+        clear: config.clear,
+        deleteMissing: config.deleteMissing,
+        transform: config.transform ?? null,
+        renameCollection: config.renameCollection ?? {},
+        idPrefix: config.idPrefix ?? null,
+        idSuffix: config.idSuffix ?? null,
+        webhook: config.webhook ?? null,
+    };
+}
+
+export function loadConfigFile(configPath?: string): Partial<Config> {
+    if (!configPath) return {};
+
+    const absolutePath = path.resolve(configPath);
+    if (!fs.existsSync(absolutePath)) {
+        throw new Error(`Config file not found: ${absolutePath}`);
+    }
+
+    const content = fs.readFileSync(absolutePath, 'utf-8');
+    const format = getFileFormat(absolutePath);
+
+    console.log(`📄 Loaded config from: ${absolutePath} (${format.toUpperCase()})\n`);
+
+    return format === 'json' ? parseJsonConfig(content) : parseIniConfig(content);
+}
+
+export function mergeConfig(
+    defaultConfig: Config,
+    fileConfig: Partial<Config>,
+    cliArgs: CliArgs
+): Config {
+    const cliWhereFilters = parseWhereFilters(cliArgs.where);
+    const cliRenameCollection = parseRenameMapping(cliArgs.renameCollection);
+
+    return {
+        collections: cliArgs.collections ?? fileConfig.collections ?? defaultConfig.collections,
+        includeSubcollections:
+            cliArgs.includeSubcollections ??
+            fileConfig.includeSubcollections ??
+            defaultConfig.includeSubcollections,
+        dryRun: cliArgs.dryRun ?? fileConfig.dryRun ?? defaultConfig.dryRun,
+        batchSize: cliArgs.batchSize ?? fileConfig.batchSize ?? defaultConfig.batchSize,
+        limit: cliArgs.limit ?? fileConfig.limit ?? defaultConfig.limit,
+        sourceProject:
+            cliArgs.sourceProject ?? fileConfig.sourceProject ?? defaultConfig.sourceProject,
+        destProject: cliArgs.destProject ?? fileConfig.destProject ?? defaultConfig.destProject,
+        retries: cliArgs.retries ?? defaultConfig.retries,
+        where:
+            cliWhereFilters.length > 0
+                ? cliWhereFilters
+                : (fileConfig.where ?? defaultConfig.where),
+        exclude: cliArgs.exclude ?? fileConfig.exclude ?? defaultConfig.exclude,
+        merge: cliArgs.merge ?? fileConfig.merge ?? defaultConfig.merge,
+        parallel: cliArgs.parallel ?? fileConfig.parallel ?? defaultConfig.parallel,
+        clear: cliArgs.clear ?? fileConfig.clear ?? defaultConfig.clear,
+        deleteMissing:
+            cliArgs.deleteMissing ?? fileConfig.deleteMissing ?? defaultConfig.deleteMissing,
+        transform: cliArgs.transform ?? fileConfig.transform ?? defaultConfig.transform,
+        renameCollection:
+            Object.keys(cliRenameCollection).length > 0
+                ? cliRenameCollection
+                : (fileConfig.renameCollection ?? defaultConfig.renameCollection),
+        idPrefix: cliArgs.idPrefix ?? fileConfig.idPrefix ?? defaultConfig.idPrefix,
+        idSuffix: cliArgs.idSuffix ?? fileConfig.idSuffix ?? defaultConfig.idSuffix,
+        webhook: cliArgs.webhook ?? fileConfig.webhook ?? defaultConfig.webhook,
+        resume: cliArgs.resume ?? defaultConfig.resume,
+        stateFile: cliArgs.stateFile ?? defaultConfig.stateFile,
+        verify: cliArgs.verify ?? defaultConfig.verify,
+        rateLimit: cliArgs.rateLimit ?? defaultConfig.rateLimit,
+        skipOversized: cliArgs.skipOversized ?? defaultConfig.skipOversized,
+        json: cliArgs.json ?? defaultConfig.json,
+    };
+}

package/src/config/validator.ts

@@ -0,0 +1,29 @@
+import type { Config } from '../types.js';
+
+export function validateConfig(config: Config): string[] {
+    const errors: string[] = [];
+
+    if (!config.sourceProject) {
+        errors.push('Source project is required (--source-project or in config file)');
+    }
+    if (!config.destProject) {
+        errors.push('Destination project is required (--dest-project or in config file)');
+    }
+    if (config.sourceProject && config.destProject && config.sourceProject === config.destProject) {
+        // Same project is allowed only if we're renaming collections or modifying IDs
+        const hasRenamedCollections = Object.keys(config.renameCollection).length > 0;
+        const hasIdModification = config.idPrefix !== null || config.idSuffix !== null;
+
+        if (!hasRenamedCollections && !hasIdModification) {
+            errors.push(
+                'Source and destination projects are the same. ' +
+                    'Use --rename-collection or --id-prefix/--id-suffix to avoid overwriting data.'
+            );
+        }
+    }
+    if (!config.collections || config.collections.length === 0) {
+        errors.push('At least one collection is required (-c or --collections)');
+    }
+
+    return errors;
+}

package/src/interactive.ts

@@ -0,0 +1,172 @@
+import admin from 'firebase-admin';
+import type { Firestore } from 'firebase-admin/firestore';
+import { input, checkbox, confirm } from '@inquirer/prompts';
+import type { Config } from './types.js';
+
+export async function runInteractiveMode(config: Config): Promise<Config> {
+    console.log('\n' + '='.repeat(60));
+    console.log('🔄 FSCOPY - INTERACTIVE MODE');
+    console.log('='.repeat(60) + '\n');
+
+    // Prompt for source project if not set
+    let sourceProject = config.sourceProject;
+    if (!sourceProject) {
+        sourceProject = await input({
+            message: 'Source Firebase project ID:',
+            validate: (value) => value.length > 0 || 'Project ID is required',
+        });
+    } else {
+        console.log(`📤 Source project: ${sourceProject}`);
+    }
+
+    // Prompt for destination project if not set
+    let destProject = config.destProject;
+    if (!destProject) {
+        destProject = await input({
+            message: 'Destination Firebase project ID:',
+            validate: (value) => value.length > 0 || 'Project ID is required',
+        });
+    } else {
+        console.log(`📥 Destination project: ${destProject}`);
+    }
+
+    // If source = destination, ask for rename/id modifications
+    const renameCollection = config.renameCollection;
+    let idPrefix = config.idPrefix;
+    let idSuffix = config.idSuffix;
+
+    if (sourceProject === destProject) {
+        console.log('\n⚠️  Source and destination are the same project.');
+        console.log('   You need to rename collections or modify document IDs to avoid overwriting.\n');
+
+        const modifyIds = await confirm({
+            message: 'Add a prefix to document IDs?',
+            default: true,
+        });
+
+        if (modifyIds) {
+            idPrefix = await input({
+                message: 'Document ID prefix (e.g., "backup_"):',
+                default: 'backup_',
+                validate: (value) => value.length > 0 || 'Prefix is required',
+            });
+        } else {
+            // Ask for suffix as alternative
+            const useSuffix = await confirm({
+                message: 'Add a suffix to document IDs instead?',
+                default: true,
+            });
+
+            if (useSuffix) {
+                idSuffix = await input({
+                    message: 'Document ID suffix (e.g., "_backup"):',
+                    default: '_backup',
+                    validate: (value) => value.length > 0 || 'Suffix is required',
+                });
+            } else {
+                console.log('\n❌ Cannot proceed: source and destination are the same without ID modification.');
+                console.log('   This would overwrite your data. Use --rename-collection, --id-prefix, or --id-suffix.\n');
+                process.exit(1);
+            }
+        }
+    }
+
+    // Initialize source Firebase to list collections
+    console.log('\n📊 Connecting to source project...');
+
+    let tempSourceApp: admin.app.App;
+    let sourceDb: Firestore;
+    let rootCollections: FirebaseFirestore.CollectionReference[];
+
+    try {
+        tempSourceApp = admin.initializeApp(
+            {
+                credential: admin.credential.applicationDefault(),
+                projectId: sourceProject,
+            },
+            'interactive-source'
+        );
+        sourceDb = tempSourceApp.firestore();
+
+        // List collections (also tests connectivity)
+        rootCollections = await sourceDb.listCollections();
+    } catch (error) {
+        const err = error as Error & { code?: string };
+        console.error('\n❌ Cannot connect to Firebase project:', err.message);
+
+        if (err.message.includes('default credentials') || err.message.includes('credential')) {
+            console.error('\n   Run this command to authenticate:');
+            console.error('   gcloud auth application-default login\n');
+        } else if (err.message.includes('not found') || err.message.includes('NOT_FOUND')) {
+            console.error(`\n   Project "${sourceProject}" not found. Check the project ID.\n`);
+        } else if (err.message.includes('permission') || err.message.includes('PERMISSION_DENIED')) {
+            console.error('\n   You don\'t have permission to access this project\'s Firestore.\n');
+        }
+
+        process.exit(1);
+    }
+
+    const collectionIds = rootCollections.map((col) => col.id);
+
+    if (collectionIds.length === 0) {
+        console.log('\n⚠️  No collections found in source project');
+        await tempSourceApp.delete();
+        process.exit(0);
+    }
+
+    // Count documents in each collection for preview
+    console.log('\n📋 Available collections:');
+    const collectionInfo: { id: string; count: number }[] = [];
+    for (const id of collectionIds) {
+        const snapshot = await sourceDb.collection(id).count().get();
+        const count = snapshot.data().count;
+        collectionInfo.push({ id, count });
+        console.log(`   - ${id} (${count} documents)`);
+    }
+
+    // Let user select collections
+    console.log('');
+    const selectedCollections = await checkbox({
+        message: 'Select collections to transfer:',
+        choices: collectionInfo.map((col) => ({
+            name: `${col.id} (${col.count} docs)`,
+            value: col.id,
+            checked: config.collections.includes(col.id),
+        })),
+        validate: (value) => value.length > 0 || 'Select at least one collection',
+    });
+
+    // Ask about options
+    console.log('');
+    const includeSubcollections = await confirm({
+        message: 'Include subcollections?',
+        default: config.includeSubcollections,
+    });
+
+    const dryRun = await confirm({
+        message: 'Dry run mode (preview without writing)?',
+        default: config.dryRun,
+    });
+
+    const merge = await confirm({
+        message: 'Merge mode (update instead of overwrite)?',
+        default: config.merge,
+    });
+
+    // Clean up temporary app
+    await tempSourceApp.delete();
+
+    // Return updated config
+    return {
+        ...config,
+        sourceProject,
+        destProject,
+        collections: selectedCollections,
+        includeSubcollections,
+        dryRun,
+        merge,
+        renameCollection,
+        idPrefix,
+        idSuffix,
+    };
+}