@fazetitans/fscopy 1.1.1 → 1.1.3

package/README.md

@@ -237,6 +237,8 @@ The transform function receives:
 
 Return the transformed document, or `null` to skip it.
 
+> **Security Warning**: The `--transform` option executes arbitrary code from the specified file. Only use transform files from trusted sources. Never run transforms from untrusted or unverified files as they have full access to your system.
+
 ### Webhook Notifications
 
 Get notified when transfers complete (success or failure):
@@ -393,6 +395,14 @@ fscopy --init config.json
 4. **Retry logic** - Automatic retry with exponential backoff on failures
 5. **Subcollection discovery** - Uses `listCollections()` to find nested data
 
+## Security
+
+- **Transform files execute arbitrary code** - The `--transform` option uses dynamic imports to load and execute JavaScript/TypeScript files. Only use transform files you have written or thoroughly reviewed. Malicious transform files could access your filesystem, network, or credentials.
+
+- **Webhook URLs should use HTTPS** - fscopy warns if you use HTTP webhooks (except localhost). Webhook payloads contain project names and transfer statistics that could be sensitive.
+
+- **Credentials via ADC** - fscopy uses Google Application Default Credentials. Ensure you're authenticated with the correct account before running transfers.
+
 ## Notes
 
 - **Dry run is ON by default** - Use `-d false` for actual transfer

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@fazetitans/fscopy",
-    "version": "1.1.1",
+    "version": "1.1.3",
     "description": "Fast CLI tool to copy Firestore collections between Firebase projects with filtering, parallel transfers, and subcollection support",
     "type": "module",
     "bin": {
@@ -46,7 +46,8 @@
         "node": ">=18.0.0"
     },
     "files": [
-        "src/cli.ts",
+        "src/**/*.ts",
+        "!src/__tests__/**",
         "README.md",
         "LICENSE"
     ],

package/src/cli.ts

@@ -23,7 +23,7 @@ import { validateConfig } from './config/validator.js';
 import { defaults } from './config/defaults.js';
 import { generateConfigFile } from './config/generator.js';
 import { loadTransferState, saveTransferState, createInitialState, validateStateForResume, deleteTransferState } from './state/index.js';
-import { sendWebhook } from './webhook/index.js';
+import { sendWebhook, validateWebhookUrl } from './webhook/index.js';
 import { countDocuments, transferCollection, clearCollection, deleteOrphanDocuments, processInParallel, getDestCollectionPath, type TransferContext, type CountProgress } from './transfer/index.js';
 import { runInteractiveMode } from './interactive.js';
 
@@ -442,6 +442,18 @@ try {
         process.exit(1);
     }
 
+    // Validate webhook URL if configured
+    if (config.webhook) {
+        const webhookValidation = validateWebhookUrl(config.webhook);
+        if (!webhookValidation.valid) {
+            console.log(`\n❌ ${webhookValidation.warning}`);
+            process.exit(1);
+        }
+        if (webhookValidation.warning) {
+            console.log(`\n⚠️  ${webhookValidation.warning}`);
+        }
+    }
+
     // Skip confirmation in interactive mode (already confirmed by selection)
     if (!argv.yes && !argv.interactive) {
         const confirmed = await askConfirmation(config);

package/src/config/defaults.ts

@@ -0,0 +1,88 @@
+import type { Config } from '../types.js';
+
+export const defaults: Config = {
+    collections: [],
+    includeSubcollections: false,
+    dryRun: true,
+    batchSize: 500,
+    limit: 0,
+    sourceProject: null,
+    destProject: null,
+    retries: 3,
+    where: [],
+    exclude: [],
+    merge: false,
+    parallel: 1,
+    clear: false,
+    deleteMissing: false,
+    transform: null,
+    renameCollection: {},
+    idPrefix: null,
+    idSuffix: null,
+    webhook: null,
+    resume: false,
+    stateFile: '.fscopy-state.json',
+    verify: false,
+    rateLimit: 0,
+    skipOversized: false,
+    json: false,
+};
+
+export const iniTemplate = `; fscopy configuration file
+
+[projects]
+source = my-source-project
+dest = my-dest-project
+
+[transfer]
+; Comma-separated list of collections
+collections = collection1, collection2
+includeSubcollections = false
+dryRun = true
+batchSize = 500
+limit = 0
+
+[options]
+; Filter documents: "field operator value" (operators: ==, !=, <, >, <=, >=)
+; where = status == active
+; Exclude subcollections by pattern (comma-separated, supports glob)
+; exclude = logs, temp/*, cache
+; Merge documents instead of overwriting
+merge = false
+; Number of parallel collection transfers
+parallel = 1
+; Clear destination collections before transfer (DESTRUCTIVE)
+clear = false
+; Delete destination docs not present in source (sync mode)
+deleteMissing = false
+; Transform documents during transfer (path to JS/TS file)
+; transform = ./transforms/anonymize.ts
+; Rename collections in destination (format: source:dest, comma-separated)
+; renameCollection = users:users_backup, orders:orders_2024
+; Add prefix or suffix to document IDs
+; idPrefix = backup_
+; idSuffix = _v2
+; Webhook URL for transfer notifications (Slack, Discord, or custom)
+; webhook = https://hooks.slack.com/services/...
+`;
+
+export const jsonTemplate = {
+    sourceProject: 'my-source-project',
+    destProject: 'my-dest-project',
+    collections: ['collection1', 'collection2'],
+    includeSubcollections: false,
+    dryRun: true,
+    batchSize: 500,
+    limit: 0,
+    where: [],
+    exclude: [],
+    merge: false,
+    parallel: 1,
+    clear: false,
+    deleteMissing: false,
+    transform: null,
+    renameCollection: {},
+    idPrefix: null,
+    idSuffix: null,
+    webhook: null,
+};

package/src/config/generator.ts

@@ -0,0 +1,27 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { getFileFormat } from './parser.js';
+import { iniTemplate, jsonTemplate } from './defaults.js';
+
+export function generateConfigFile(outputPath: string): boolean {
+    const filePath = path.resolve(outputPath);
+    const format = getFileFormat(filePath);
+
+    if (fs.existsSync(filePath)) {
+        console.error(`❌ File already exists: ${filePath}`);
+        console.error('   Use a different filename or delete the existing file.');
+        process.exitCode = 1;
+        return false;
+    }
+
+    const content = format === 'json' ? JSON.stringify(jsonTemplate, null, 4) : iniTemplate;
+
+    fs.writeFileSync(filePath, content, 'utf-8');
+
+    console.log(`✓ Config template created: ${filePath}`);
+    console.log('');
+    console.log('Edit the file to configure your transfer, then run:');
+    console.log(`  fscopy -f ${outputPath}`);
+
+    return true;
+}

package/src/config/index.ts

@@ -0,0 +1,15 @@
+export { defaults, iniTemplate, jsonTemplate } from './defaults.js';
+export {
+    getFileFormat,
+    parseBoolean,
+    parseWhereFilter,
+    parseWhereFilters,
+    parseStringList,
+    parseRenameMapping,
+    parseIniConfig,
+    parseJsonConfig,
+    loadConfigFile,
+    mergeConfig,
+} from './parser.js';
+export { validateConfig } from './validator.js';
+export { generateConfigFile } from './generator.js';

package/src/config/parser.ts

@@ -0,0 +1,261 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import ini from 'ini';
+import type { Config, WhereFilter, CliArgs } from '../types.js';
+
+export function getFileFormat(filePath: string): 'json' | 'ini' {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext === '.json') return 'json';
+    return 'ini';
+}
+
+export function parseBoolean(val: unknown): boolean {
+    if (typeof val === 'boolean') return val;
+    if (typeof val === 'string') {
+        return val.toLowerCase() === 'true';
+    }
+    return false;
+}
+
+export function parseWhereFilter(filterStr: string): WhereFilter | null {
+    const operatorRegex = /(==|!=|<=|>=|<|>)/;
+    const match = new RegExp(operatorRegex).exec(filterStr);
+
+    if (!match) {
+        console.warn(`⚠️  Invalid where filter: "${filterStr}" (missing operator)`);
+        return null;
+    }
+
+    const operator = match[0] as FirebaseFirestore.WhereFilterOp;
+    const [fieldPart, valuePart] = filterStr.split(operatorRegex).filter((_, i) => i !== 1);
+
+    if (!fieldPart || !valuePart) {
+        console.warn(`⚠️  Invalid where filter: "${filterStr}" (missing field or value)`);
+        return null;
+    }
+
+    const field = fieldPart.trim();
+    const rawValue = valuePart.trim();
+
+    let value: string | number | boolean;
+    if (rawValue === 'true') {
+        value = true;
+    } else if (rawValue === 'false') {
+        value = false;
+    } else if (rawValue === 'null') {
+        value = null as unknown as string;
+    } else if (!Number.isNaN(Number(rawValue)) && rawValue !== '') {
+        value = Number(rawValue);
+    } else {
+        value = rawValue.replaceAll(/(?:^["'])|(?:["']$)/g, '');
+    }
+
+    return { field, operator, value };
+}
+
+export function parseWhereFilters(filters: string[] | undefined): WhereFilter[] {
+    if (!filters || filters.length === 0) return [];
+    return filters.map(parseWhereFilter).filter((f): f is WhereFilter => f !== null);
+}
+
+export function parseStringList(value: string | undefined): string[] {
+    if (!value) return [];
+    return value
+        .split(',')
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+
+export function parseRenameMapping(
+    mappings: string[] | string | undefined
+): Record<string, string> {
+    if (!mappings) return {};
+
+    const result: Record<string, string> = {};
+    const items = Array.isArray(mappings) ? mappings : parseStringList(mappings);
+
+    for (const item of items) {
+        const mapping = String(item).trim();
+        const colonIndex = mapping.indexOf(':');
+        if (colonIndex === -1) {
+            console.warn(`⚠️  Invalid rename mapping: "${mapping}" (missing ':')`);
+            continue;
+        }
+        const source = mapping.slice(0, colonIndex).trim();
+        const dest = mapping.slice(colonIndex + 1).trim();
+        if (!source || !dest) {
+            console.warn(`⚠️  Invalid rename mapping: "${mapping}" (empty source or dest)`);
+            continue;
+        }
+        result[source] = dest;
+    }
+
+    return result;
+}
+
+export function parseIniConfig(content: string): Partial<Config> {
+    const parsed = ini.parse(content) as {
+        projects?: { source?: string; dest?: string };
+        transfer?: {
+            collections?: string;
+            includeSubcollections?: string | boolean;
+            dryRun?: string | boolean;
+            batchSize?: string;
+            limit?: string;
+        };
+        options?: {
+            where?: string;
+            exclude?: string;
+            merge?: string | boolean;
+            parallel?: string;
+            clear?: string | boolean;
+            deleteMissing?: string | boolean;
+            transform?: string;
+            renameCollection?: string;
+            idPrefix?: string;
+            idSuffix?: string;
+            webhook?: string;
+        };
+    };
+
+    let collections: string[] = [];
+    if (parsed.transfer?.collections) {
+        collections = parsed.transfer.collections
+            .split(',')
+            .map((c) => c.trim())
+            .filter((c) => c.length > 0);
+    }
+
+    const whereFilters = parsed.options?.where
+        ? parseWhereFilters(parseStringList(parsed.options.where))
+        : [];
+
+    return {
+        sourceProject: parsed.projects?.source ?? null,
+        destProject: parsed.projects?.dest ?? null,
+        collections,
+        includeSubcollections: parseBoolean(parsed.transfer?.includeSubcollections),
+        dryRun: parseBoolean(parsed.transfer?.dryRun ?? 'true'),
+        batchSize: Number.parseInt(parsed.transfer?.batchSize ?? '', 10) || 500,
+        limit: Number.parseInt(parsed.transfer?.limit ?? '', 10) || 0,
+        where: whereFilters,
+        exclude: parseStringList(parsed.options?.exclude),
+        merge: parseBoolean(parsed.options?.merge),
+        parallel: Number.parseInt(parsed.options?.parallel ?? '', 10) || 1,
+        clear: parseBoolean(parsed.options?.clear),
+        deleteMissing: parseBoolean(parsed.options?.deleteMissing),
+        transform: parsed.options?.transform ?? null,
+        renameCollection: parseRenameMapping(parsed.options?.renameCollection),
+        idPrefix: parsed.options?.idPrefix ?? null,
+        idSuffix: parsed.options?.idSuffix ?? null,
+        webhook: parsed.options?.webhook ?? null,
+    };
+}
+
+export function parseJsonConfig(content: string): Partial<Config> {
+    const config = JSON.parse(content) as {
+        sourceProject?: string;
+        destProject?: string;
+        collections?: string[];
+        includeSubcollections?: boolean;
+        dryRun?: boolean;
+        batchSize?: number;
+        limit?: number;
+        where?: string[];
+        exclude?: string[];
+        merge?: boolean;
+        parallel?: number;
+        clear?: boolean;
+        deleteMissing?: boolean;
+        transform?: string;
+        renameCollection?: Record<string, string>;
+        idPrefix?: string;
+        idSuffix?: string;
+        webhook?: string;
+    };
+
+    return {
+        sourceProject: config.sourceProject ?? null,
+        destProject: config.destProject ?? null,
+        collections: config.collections,
+        includeSubcollections: config.includeSubcollections,
+        dryRun: config.dryRun,
+        batchSize: config.batchSize,
+        limit: config.limit,
+        where: parseWhereFilters(config.where),
+        exclude: config.exclude,
+        merge: config.merge,
+        parallel: config.parallel,
+        clear: config.clear,
+        deleteMissing: config.deleteMissing,
+        transform: config.transform ?? null,
+        renameCollection: config.renameCollection ?? {},
+        idPrefix: config.idPrefix ?? null,
+        idSuffix: config.idSuffix ?? null,
+        webhook: config.webhook ?? null,
+    };
+}
+
+export function loadConfigFile(configPath?: string): Partial<Config> {
+    if (!configPath) return {};
+
+    const absolutePath = path.resolve(configPath);
+    if (!fs.existsSync(absolutePath)) {
+        throw new Error(`Config file not found: ${absolutePath}`);
+    }
+
+    const content = fs.readFileSync(absolutePath, 'utf-8');
+    const format = getFileFormat(absolutePath);
+
+    console.log(`📄 Loaded config from: ${absolutePath} (${format.toUpperCase()})\n`);
+
+    return format === 'json' ? parseJsonConfig(content) : parseIniConfig(content);
+}
+
+export function mergeConfig(
+    defaultConfig: Config,
+    fileConfig: Partial<Config>,
+    cliArgs: CliArgs
+): Config {
+    const cliWhereFilters = parseWhereFilters(cliArgs.where);
+    const cliRenameCollection = parseRenameMapping(cliArgs.renameCollection);
+
+    return {
+        collections: cliArgs.collections ?? fileConfig.collections ?? defaultConfig.collections,
+        includeSubcollections:
+            cliArgs.includeSubcollections ??
+            fileConfig.includeSubcollections ??
+            defaultConfig.includeSubcollections,
+        dryRun: cliArgs.dryRun ?? fileConfig.dryRun ?? defaultConfig.dryRun,
+        batchSize: cliArgs.batchSize ?? fileConfig.batchSize ?? defaultConfig.batchSize,
+        limit: cliArgs.limit ?? fileConfig.limit ?? defaultConfig.limit,
+        sourceProject:
+            cliArgs.sourceProject ?? fileConfig.sourceProject ?? defaultConfig.sourceProject,
+        destProject: cliArgs.destProject ?? fileConfig.destProject ?? defaultConfig.destProject,
+        retries: cliArgs.retries ?? defaultConfig.retries,
+        where:
+            cliWhereFilters.length > 0
+                ? cliWhereFilters
+                : (fileConfig.where ?? defaultConfig.where),
+        exclude: cliArgs.exclude ?? fileConfig.exclude ?? defaultConfig.exclude,
+        merge: cliArgs.merge ?? fileConfig.merge ?? defaultConfig.merge,
+        parallel: cliArgs.parallel ?? fileConfig.parallel ?? defaultConfig.parallel,
+        clear: cliArgs.clear ?? fileConfig.clear ?? defaultConfig.clear,
+        deleteMissing:
+            cliArgs.deleteMissing ?? fileConfig.deleteMissing ?? defaultConfig.deleteMissing,
+        transform: cliArgs.transform ?? fileConfig.transform ?? defaultConfig.transform,
+        renameCollection:
+            Object.keys(cliRenameCollection).length > 0
+                ? cliRenameCollection
+                : (fileConfig.renameCollection ?? defaultConfig.renameCollection),
+        idPrefix: cliArgs.idPrefix ?? fileConfig.idPrefix ?? defaultConfig.idPrefix,
+        idSuffix: cliArgs.idSuffix ?? fileConfig.idSuffix ?? defaultConfig.idSuffix,
+        webhook: cliArgs.webhook ?? fileConfig.webhook ?? defaultConfig.webhook,
+        resume: cliArgs.resume ?? defaultConfig.resume,
+        stateFile: cliArgs.stateFile ?? defaultConfig.stateFile,
+        verify: cliArgs.verify ?? defaultConfig.verify,
+        rateLimit: cliArgs.rateLimit ?? defaultConfig.rateLimit,
+        skipOversized: cliArgs.skipOversized ?? defaultConfig.skipOversized,
+        json: cliArgs.json ?? defaultConfig.json,
+    };
+}

package/src/config/validator.ts

@@ -0,0 +1,29 @@
+import type { Config } from '../types.js';
+
+export function validateConfig(config: Config): string[] {
+    const errors: string[] = [];
+
+    if (!config.sourceProject) {
+        errors.push('Source project is required (--source-project or in config file)');
+    }
+    if (!config.destProject) {
+        errors.push('Destination project is required (--dest-project or in config file)');
+    }
+    if (config.sourceProject && config.destProject && config.sourceProject === config.destProject) {
+        // Same project is allowed only if we're renaming collections or modifying IDs
+        const hasRenamedCollections = Object.keys(config.renameCollection).length > 0;
+        const hasIdModification = config.idPrefix !== null || config.idSuffix !== null;
+
+        if (!hasRenamedCollections && !hasIdModification) {
+            errors.push(
+                'Source and destination projects are the same. ' +
+                    'Use --rename-collection or --id-prefix/--id-suffix to avoid overwriting data.'
+            );
+        }
+    }
+    if (!config.collections || config.collections.length === 0) {
+        errors.push('At least one collection is required (-c or --collections)');
+    }
+
+    return errors;
+}

package/src/interactive.ts

@@ -0,0 +1,172 @@
+import admin from 'firebase-admin';
+import type { Firestore } from 'firebase-admin/firestore';
+import { input, checkbox, confirm } from '@inquirer/prompts';
+import type { Config } from './types.js';
+
+export async function runInteractiveMode(config: Config): Promise<Config> {
+    console.log('\n' + '='.repeat(60));
+    console.log('🔄 FSCOPY - INTERACTIVE MODE');
+    console.log('='.repeat(60) + '\n');
+
+    // Prompt for source project if not set
+    let sourceProject = config.sourceProject;
+    if (!sourceProject) {
+        sourceProject = await input({
+            message: 'Source Firebase project ID:',
+            validate: (value) => value.length > 0 || 'Project ID is required',
+        });
+    } else {
+        console.log(`📤 Source project: ${sourceProject}`);
+    }
+
+    // Prompt for destination project if not set
+    let destProject = config.destProject;
+    if (!destProject) {
+        destProject = await input({
+            message: 'Destination Firebase project ID:',
+            validate: (value) => value.length > 0 || 'Project ID is required',
+        });
+    } else {
+        console.log(`📥 Destination project: ${destProject}`);
+    }
+
+    // If source = destination, ask for rename/id modifications
+    const renameCollection = config.renameCollection;
+    let idPrefix = config.idPrefix;
+    let idSuffix = config.idSuffix;
+
+    if (sourceProject === destProject) {
+        console.log('\n⚠️  Source and destination are the same project.');
+        console.log('   You need to rename collections or modify document IDs to avoid overwriting.\n');
+
+        const modifyIds = await confirm({
+            message: 'Add a prefix to document IDs?',
+            default: true,
+        });
+
+        if (modifyIds) {
+            idPrefix = await input({
+                message: 'Document ID prefix (e.g., "backup_"):',
+                default: 'backup_',
+                validate: (value) => value.length > 0 || 'Prefix is required',
+            });
+        } else {
+            // Ask for suffix as alternative
+            const useSuffix = await confirm({
+                message: 'Add a suffix to document IDs instead?',
+                default: true,
+            });
+
+            if (useSuffix) {
+                idSuffix = await input({
+                    message: 'Document ID suffix (e.g., "_backup"):',
+                    default: '_backup',
+                    validate: (value) => value.length > 0 || 'Suffix is required',
+                });
+            } else {
+                console.log('\n❌ Cannot proceed: source and destination are the same without ID modification.');
+                console.log('   This would overwrite your data. Use --rename-collection, --id-prefix, or --id-suffix.\n');
+                process.exit(1);
+            }
+        }
+    }
+
+    // Initialize source Firebase to list collections
+    console.log('\n📊 Connecting to source project...');
+
+    let tempSourceApp: admin.app.App;
+    let sourceDb: Firestore;
+    let rootCollections: FirebaseFirestore.CollectionReference[];
+
+    try {
+        tempSourceApp = admin.initializeApp(
+            {
+                credential: admin.credential.applicationDefault(),
+                projectId: sourceProject,
+            },
+            'interactive-source'
+        );
+        sourceDb = tempSourceApp.firestore();
+
+        // List collections (also tests connectivity)
+        rootCollections = await sourceDb.listCollections();
+    } catch (error) {
+        const err = error as Error & { code?: string };
+        console.error('\n❌ Cannot connect to Firebase project:', err.message);
+
+        if (err.message.includes('default credentials') || err.message.includes('credential')) {
+            console.error('\n   Run this command to authenticate:');
+            console.error('   gcloud auth application-default login\n');
+        } else if (err.message.includes('not found') || err.message.includes('NOT_FOUND')) {
+            console.error(`\n   Project "${sourceProject}" not found. Check the project ID.\n`);
+        } else if (err.message.includes('permission') || err.message.includes('PERMISSION_DENIED')) {
+            console.error('\n   You don\'t have permission to access this project\'s Firestore.\n');
+        }
+
+        process.exit(1);
+    }
+
+    const collectionIds = rootCollections.map((col) => col.id);
+
+    if (collectionIds.length === 0) {
+        console.log('\n⚠️  No collections found in source project');
+        await tempSourceApp.delete();
+        process.exit(0);
+    }
+
+    // Count documents in each collection for preview
+    console.log('\n📋 Available collections:');
+    const collectionInfo: { id: string; count: number }[] = [];
+    for (const id of collectionIds) {
+        const snapshot = await sourceDb.collection(id).count().get();
+        const count = snapshot.data().count;
+        collectionInfo.push({ id, count });
+        console.log(`   - ${id} (${count} documents)`);
+    }
+
+    // Let user select collections
+    console.log('');
+    const selectedCollections = await checkbox({
+        message: 'Select collections to transfer:',
+        choices: collectionInfo.map((col) => ({
+            name: `${col.id} (${col.count} docs)`,
+            value: col.id,
+            checked: config.collections.includes(col.id),
+        })),
+        validate: (value) => value.length > 0 || 'Select at least one collection',
+    });
+
+    // Ask about options
+    console.log('');
+    const includeSubcollections = await confirm({
+        message: 'Include subcollections?',
+        default: config.includeSubcollections,
+    });
+
+    const dryRun = await confirm({
+        message: 'Dry run mode (preview without writing)?',
+        default: config.dryRun,
+    });
+
+    const merge = await confirm({
+        message: 'Merge mode (update instead of overwrite)?',
+        default: config.merge,
+    });
+
+    // Clean up temporary app
+    await tempSourceApp.delete();
+
+    // Return updated config
+    return {
+        ...config,
+        sourceProject,
+        destProject,
+        collections: selectedCollections,
+        includeSubcollections,
+        dryRun,
+        merge,
+        renameCollection,
+        idPrefix,
+        idSuffix,
+    };
+}