@fazetitans/fscopy 1.1.0 → 1.1.2

package/src/state/index.ts

@@ -0,0 +1,123 @@
+import fs from 'node:fs';
+import type { Config, TransferState } from '../types.js';
+
+export const STATE_VERSION = 1;
+
+export function loadTransferState(stateFile: string): TransferState | null {
+    try {
+        if (!fs.existsSync(stateFile)) {
+            return null;
+        }
+        const content = fs.readFileSync(stateFile, 'utf-8');
+        const state = JSON.parse(content) as TransferState;
+
+        if (state.version !== STATE_VERSION) {
+            console.warn(
+                `⚠️  State file version mismatch (expected ${STATE_VERSION}, got ${state.version})`
+            );
+            return null;
+        }
+
+        return state;
+    } catch (error) {
+        console.error(`⚠️  Failed to load state file: ${(error as Error).message}`);
+        return null;
+    }
+}
+
+export function saveTransferState(stateFile: string, state: TransferState): void {
+    state.updatedAt = new Date().toISOString();
+    const content = JSON.stringify(state, null, 2);
+    const tempFile = `${stateFile}.tmp`;
+
+    try {
+        // Write to temp file first (atomic write pattern)
+        fs.writeFileSync(tempFile, content);
+        // Rename is atomic on most filesystems
+        fs.renameSync(tempFile, stateFile);
+    } catch (error) {
+        // Clean up temp file if it exists
+        try {
+            if (fs.existsSync(tempFile)) {
+                fs.unlinkSync(tempFile);
+            }
+        } catch {
+            // Ignore cleanup errors
+        }
+        // Log but don't throw - state save failure shouldn't stop the transfer
+        console.error(`⚠️  Failed to save state file: ${(error as Error).message}`);
+    }
+}
+
+export function deleteTransferState(stateFile: string): void {
+    try {
+        if (fs.existsSync(stateFile)) {
+            fs.unlinkSync(stateFile);
+        }
+    } catch {
+        // Ignore errors when deleting state file
+    }
+}
+
+export function createInitialState(config: Config): TransferState {
+    return {
+        version: STATE_VERSION,
+        sourceProject: config.sourceProject!,
+        destProject: config.destProject!,
+        collections: config.collections,
+        startedAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+        completedDocs: {},
+        stats: {
+            collectionsProcessed: 0,
+            documentsTransferred: 0,
+            documentsDeleted: 0,
+            errors: 0,
+        },
+    };
+}
+
+export function validateStateForResume(state: TransferState, config: Config): string[] {
+    const errors: string[] = [];
+
+    if (state.sourceProject !== config.sourceProject) {
+        errors.push(
+            `Source project mismatch: state has "${state.sourceProject}", config has "${config.sourceProject}"`
+        );
+    }
+    if (state.destProject !== config.destProject) {
+        errors.push(
+            `Destination project mismatch: state has "${state.destProject}", config has "${config.destProject}"`
+        );
+    }
+
+    // Check if collections are compatible (state collections should be subset of config)
+    const configCollections = new Set(config.collections);
+    for (const col of state.collections) {
+        if (!configCollections.has(col)) {
+            errors.push(`State contains collection "${col}" not in current config`);
+        }
+    }
+
+    return errors;
+}
+
+export function isDocCompleted(
+    state: TransferState,
+    collectionPath: string,
+    docId: string
+): boolean {
+    const completedInCollection = state.completedDocs[collectionPath];
+    return completedInCollection ? completedInCollection.includes(docId) : false;
+}
+
+export function markDocCompleted(
+    state: TransferState,
+    collectionPath: string,
+    docId: string
+): void {
+    if (!state.completedDocs[collectionPath]) {
+        state.completedDocs[collectionPath] = [];
+    }
+    state.completedDocs[collectionPath].push(docId);
+}

package/src/transfer/clear.ts

@@ -0,0 +1,157 @@
+import type { Firestore } from 'firebase-admin/firestore';
+import type { Config } from '../types.js';
+import type { Logger } from '../utils/logger.js';
+import { withRetry } from '../utils/retry.js';
+import { matchesExcludePattern } from '../utils/patterns.js';
+import { getSubcollections, getDestCollectionPath } from './helpers.js';
+
+export async function clearCollection(
+    db: Firestore,
+    collectionPath: string,
+    config: Config,
+    logger: Logger,
+    includeSubcollections: boolean
+): Promise<number> {
+    let deletedCount = 0;
+    const collectionRef = db.collection(collectionPath);
+    const snapshot = await collectionRef.get();
+
+    if (snapshot.empty) {
+        return 0;
+    }
+
+    // Delete subcollections first if enabled
+    if (includeSubcollections) {
+        for (const doc of snapshot.docs) {
+            const subcollections = await getSubcollections(doc.ref);
+            for (const subId of subcollections) {
+                // Check exclude patterns
+                if (matchesExcludePattern(subId, config.exclude)) {
+                    continue;
+                }
+                const subPath = `${collectionPath}/${doc.id}/${subId}`;
+                deletedCount += await clearCollection(db, subPath, config, logger, true);
+            }
+        }
+    }
+
+    // Delete documents in batches
+    const docs = snapshot.docs;
+    for (let i = 0; i < docs.length; i += config.batchSize) {
+        const batch = docs.slice(i, i + config.batchSize);
+        const writeBatch = db.batch();
+
+        for (const doc of batch) {
+            writeBatch.delete(doc.ref);
+            deletedCount++;
+        }
+
+        if (!config.dryRun) {
+            await withRetry(() => writeBatch.commit(), {
+                retries: config.retries,
+                onRetry: (attempt, max, err, delay) => {
+                    logger.error(`Retry delete ${attempt}/${max} for ${collectionPath}`, {
+                        error: err.message,
+                        delay,
+                    });
+                },
+            });
+        }
+
+        logger.info(`Deleted ${batch.length} documents from ${collectionPath}`);
+    }
+
+    return deletedCount;
+}
+
+export async function deleteOrphanDocuments(
+    sourceDb: Firestore,
+    destDb: Firestore,
+    sourceCollectionPath: string,
+    config: Config,
+    logger: Logger
+): Promise<number> {
+    let deletedCount = 0;
+
+    // Get the destination path (may be renamed)
+    const destCollectionPath = getDestCollectionPath(sourceCollectionPath, config.renameCollection);
+
+    // Get all document IDs from source (use select() to only fetch IDs, not data)
+    const sourceSnapshot = await sourceDb.collection(sourceCollectionPath).select().get();
+    const sourceIds = new Set(sourceSnapshot.docs.map((doc) => doc.id));
+
+    // Get all document IDs from destination (use select() to only fetch IDs, not data)
+    const destSnapshot = await destDb.collection(destCollectionPath).select().get();
+
+    // Find orphan documents (in dest but not in source)
+    const orphanDocs = destSnapshot.docs.filter((doc) => !sourceIds.has(doc.id));
+
+    if (orphanDocs.length === 0) {
+        return 0;
+    }
+
+    logger.info(`Found ${orphanDocs.length} orphan documents in ${destCollectionPath}`);
+
+    // Delete orphan documents in batches
+    for (let i = 0; i < orphanDocs.length; i += config.batchSize) {
+        const batch = orphanDocs.slice(i, i + config.batchSize);
+        const writeBatch = destDb.batch();
+
+        for (const doc of batch) {
+            // If subcollections are included, recursively delete orphans in subcollections first
+            if (config.includeSubcollections) {
+                const subcollections = await getSubcollections(doc.ref);
+                for (const subId of subcollections) {
+                    if (matchesExcludePattern(subId, config.exclude)) {
+                        continue;
+                    }
+                    const subPath = `${destCollectionPath}/${doc.id}/${subId}`;
+                    // For orphan parent docs, clear all subcollection data
+                    deletedCount += await clearCollection(destDb, subPath, config, logger, true);
+                }
+            }
+
+            writeBatch.delete(doc.ref);
+            deletedCount++;
+        }
+
+        if (!config.dryRun) {
+            await withRetry(() => writeBatch.commit(), {
+                retries: config.retries,
+                onRetry: (attempt, max, err, delay) => {
+                    logger.error(
+                        `Retry delete orphans ${attempt}/${max} for ${destCollectionPath}`,
+                        {
+                            error: err.message,
+                            delay,
+                        }
+                    );
+                },
+            });
+        }
+
+        logger.info(`Deleted ${batch.length} orphan documents from ${destCollectionPath}`);
+    }
+
+    // Also check subcollections of existing documents for orphans
+    if (config.includeSubcollections) {
+        for (const sourceDoc of sourceSnapshot.docs) {
+            const sourceSubcollections = await getSubcollections(sourceDoc.ref);
+            for (const subId of sourceSubcollections) {
+                if (matchesExcludePattern(subId, config.exclude)) {
+                    continue;
+                }
+                const subPath = `${sourceCollectionPath}/${sourceDoc.id}/${subId}`;
+                deletedCount += await deleteOrphanDocuments(
+                    sourceDb,
+                    destDb,
+                    subPath,
+                    config,
+                    logger
+                );
+            }
+        }
+    }
+
+    return deletedCount;
+}

package/src/transfer/count.ts

@@ -0,0 +1,71 @@
+import type { Firestore } from 'firebase-admin/firestore';
+import type { Config } from '../types.js';
+import { matchesExcludePattern } from '../utils/patterns.js';
+import { getSubcollections } from './helpers.js';
+
+export interface CountProgress {
+    onCollection?: (path: string, count: number) => void;
+    onSubcollection?: (path: string) => void;
+}
+
+export async function countDocuments(
+    sourceDb: Firestore,
+    collectionPath: string,
+    config: Config,
+    depth: number = 0,
+    progress?: CountProgress
+): Promise<number> {
+    let count = 0;
+
+    // Build query with where filters (only at root level)
+    let query: FirebaseFirestore.Query = sourceDb.collection(collectionPath);
+    if (depth === 0 && config.where.length > 0) {
+        for (const filter of config.where) {
+            query = query.where(filter.field, filter.operator, filter.value);
+        }
+    }
+
+    // Use count() aggregation to avoid downloading all documents (much cheaper)
+    // But we need document refs for subcollections, so we'll need a different approach
+    if (config.includeSubcollections) {
+        // When including subcollections, we need to fetch docs to get their refs
+        // Use select() to only fetch document IDs, not the data (reduces bandwidth)
+        const snapshot = await query.select().get();
+        count += snapshot.size;
+
+        // Report progress for root collections
+        if (depth === 0 && progress?.onCollection) {
+            progress.onCollection(collectionPath, snapshot.size);
+        }
+
+        for (const doc of snapshot.docs) {
+            const subcollections = await getSubcollections(doc.ref);
+            for (const subId of subcollections) {
+                const subPath = `${collectionPath}/${doc.id}/${subId}`;
+
+                // Check exclude patterns
+                if (matchesExcludePattern(subId, config.exclude)) {
+                    continue;
+                }
+
+                // Report subcollection discovery
+                if (progress?.onSubcollection) {
+                    progress.onSubcollection(subPath);
+                }
+
+                count += await countDocuments(sourceDb, subPath, config, depth + 1, progress);
+            }
+        }
+    } else {
+        // No subcollections: use count() aggregation (1 read instead of N)
+        const countSnapshot = await query.count().get();
+        count = countSnapshot.data().count;
+
+        // Report progress for root collections
+        if (depth === 0 && progress?.onCollection) {
+            progress.onCollection(collectionPath, count);
+        }
+    }
+
+    return count;
+}

package/src/transfer/helpers.ts

@@ -0,0 +1,37 @@
+import type { DocumentReference } from 'firebase-admin/firestore';
+
+export async function getSubcollections(docRef: DocumentReference): Promise<string[]> {
+    const collections = await docRef.listCollections();
+    return collections.map((col) => col.id);
+}
+
+export function getDestCollectionPath(
+    sourcePath: string,
+    renameMapping: Record<string, string>
+): string {
+    // Get the root collection name from the source path
+    const rootCollection = sourcePath.split('/')[0];
+
+    // Check if this root collection should be renamed
+    if (renameMapping[rootCollection]) {
+        // Replace the root collection name with the destination name
+        return renameMapping[rootCollection] + sourcePath.slice(rootCollection.length);
+    }
+
+    return sourcePath;
+}
+
+export function getDestDocId(
+    sourceId: string,
+    prefix: string | null,
+    suffix: string | null
+): string {
+    let destId = sourceId;
+    if (prefix) {
+        destId = prefix + destId;
+    }
+    if (suffix) {
+        destId = destId + suffix;
+    }
+    return destId;
+}

package/src/transfer/index.ts

@@ -0,0 +1,5 @@
+export { getSubcollections, getDestCollectionPath, getDestDocId } from './helpers.js';
+export { processInParallel, type ParallelResult } from './parallel.js';
+export { countDocuments, type CountProgress } from './count.js';
+export { clearCollection, deleteOrphanDocuments } from './clear.js';
+export { transferCollection, type TransferContext } from './transfer.js';

package/src/transfer/parallel.ts

@@ -0,0 +1,51 @@
+export interface ParallelResult<R> {
+    results: R[];
+    errors: Error[];
+}
+
+export async function processInParallel<T, R>(
+    items: T[],
+    concurrency: number,
+    processor: (item: T) => Promise<R>
+): Promise<ParallelResult<R>> {
+    const results: R[] = [];
+    const errors: Error[] = [];
+    const queue = [...items];
+    const executing: Set<Promise<void>> = new Set();
+
+    const processNext = async (): Promise<void> => {
+        if (queue.length === 0) return;
+
+        const item = queue.shift()!;
+        try {
+            const result = await processor(item);
+            results.push(result);
+        } catch (error) {
+            errors.push(error instanceof Error ? error : new Error(String(error)));
+        }
+    };
+
+    // Start initial batch of concurrent tasks
+    while (executing.size < concurrency && queue.length > 0) {
+        const promise = processNext().then(() => {
+            executing.delete(promise);
+        });
+        executing.add(promise);
+    }
+
+    // Process remaining items as slots become available
+    while (queue.length > 0 || executing.size > 0) {
+        if (executing.size > 0) {
+            await Promise.race(executing);
+        }
+        // Fill up to concurrency limit
+        while (executing.size < concurrency && queue.length > 0) {
+            const promise = processNext().then(() => {
+                executing.delete(promise);
+            });
+            executing.add(promise);
+        }
+    }
+
+    return { results, errors };
+}

package/src/transfer/transfer.ts

@@ -0,0 +1,214 @@
+import type { Firestore, WriteBatch } from 'firebase-admin/firestore';
+import type cliProgress from 'cli-progress';
+import type { Config, Stats, TransferState, TransformFunction } from '../types.js';
+import type { Logger } from '../utils/logger.js';
+import type { RateLimiter } from '../utils/rate-limiter.js';
+import { withRetry } from '../utils/retry.js';
+import { matchesExcludePattern } from '../utils/patterns.js';
+import { estimateDocumentSize, formatBytes, FIRESTORE_MAX_DOC_SIZE } from '../utils/doc-size.js';
+import { isDocCompleted, markDocCompleted, saveTransferState } from '../state/index.js';
+import { getSubcollections, getDestCollectionPath, getDestDocId } from './helpers.js';
+
+export interface TransferContext {
+    sourceDb: Firestore;
+    destDb: Firestore;
+    config: Config;
+    stats: Stats;
+    logger: Logger;
+    progressBar: cliProgress.SingleBar | null;
+    transformFn: TransformFunction | null;
+    state: TransferState | null;
+    rateLimiter: RateLimiter | null;
+}
+
+export async function transferCollection(
+    ctx: TransferContext,
+    collectionPath: string,
+    depth: number = 0
+): Promise<void> {
+    const { sourceDb, destDb, config, stats, logger, progressBar, transformFn, state, rateLimiter } = ctx;
+
+    // Get the destination path (may be renamed)
+    const destCollectionPath = getDestCollectionPath(collectionPath, config.renameCollection);
+
+    const sourceCollectionRef = sourceDb.collection(collectionPath);
+    let query: FirebaseFirestore.Query = sourceCollectionRef;
+
+    // Apply where filters (only at root level)
+    if (depth === 0 && config.where.length > 0) {
+        for (const filter of config.where) {
+            query = query.where(filter.field, filter.operator, filter.value);
+        }
+    }
+
+    if (config.limit > 0 && depth === 0) {
+        query = query.limit(config.limit);
+    }
+
+    const snapshot = await withRetry(() => query.get(), {
+        retries: config.retries,
+        onRetry: (attempt, max, err, delay) => {
+            logger.error(`Retry ${attempt}/${max} for ${collectionPath}`, {
+                error: err.message,
+                delay,
+            });
+        },
+    });
+
+    if (snapshot.empty) {
+        return;
+    }
+
+    stats.collectionsProcessed++;
+    logger.info(`Processing collection: ${collectionPath}`, { documents: snapshot.size });
+
+    const docs = snapshot.docs;
+    const batchDocIds: string[] = []; // Track docs in current batch for state saving
+
+    for (let i = 0; i < docs.length; i += config.batchSize) {
+        const batch = docs.slice(i, i + config.batchSize);
+        const destBatch: WriteBatch = destDb.batch();
+        batchDocIds.length = 0; // Clear for new batch
+
+        for (const doc of batch) {
+            // Skip if already completed (resume mode)
+            if (state && isDocCompleted(state, collectionPath, doc.id)) {
+                if (progressBar) {
+                    progressBar.increment();
+                }
+                stats.documentsTransferred++;
+                continue;
+            }
+
+            // Get destination document ID (with optional prefix/suffix)
+            const destDocId = getDestDocId(doc.id, config.idPrefix, config.idSuffix);
+            const destDocRef = destDb.collection(destCollectionPath).doc(destDocId);
+
+            // Apply transform if provided
+            let docData = doc.data() as Record<string, unknown>;
+            if (transformFn) {
+                try {
+                    const transformed = transformFn(docData, {
+                        id: doc.id,
+                        path: `${collectionPath}/${doc.id}`,
+                    });
+                    if (transformed === null) {
+                        // Skip this document if transform returns null
+                        logger.info('Skipped document (transform returned null)', {
+                            collection: collectionPath,
+                            docId: doc.id,
+                        });
+                        if (progressBar) {
+                            progressBar.increment();
+                        }
+                        // Mark as completed even if skipped
+                        batchDocIds.push(doc.id);
+                        continue;
+                    }
+                    docData = transformed;
+                } catch (transformError) {
+                    const errMsg =
+                        transformError instanceof Error
+                            ? transformError.message
+                            : String(transformError);
+                    logger.error(`Transform failed for document ${doc.id}`, {
+                        collection: collectionPath,
+                        error: errMsg,
+                    });
+                    stats.errors++;
+                    if (progressBar) {
+                        progressBar.increment();
+                    }
+                    // Skip this document but continue with others
+                    continue;
+                }
+            }
+
+            // Check document size
+            const docSize = estimateDocumentSize(docData, `${destCollectionPath}/${destDocId}`);
+            if (docSize > FIRESTORE_MAX_DOC_SIZE) {
+                const sizeStr = formatBytes(docSize);
+                if (config.skipOversized) {
+                    logger.info(`Skipped oversized document (${sizeStr})`, {
+                        collection: collectionPath,
+                        docId: doc.id,
+                    });
+                    if (progressBar) {
+                        progressBar.increment();
+                    }
+                    batchDocIds.push(doc.id);
+                    continue;
+                } else {
+                    throw new Error(
+                        `Document ${collectionPath}/${doc.id} exceeds 1MB limit (${sizeStr}). Use --skip-oversized to skip.`
+                    );
+                }
+            }
+
+            if (!config.dryRun) {
+                // Use merge option if enabled
+                if (config.merge) {
+                    destBatch.set(destDocRef, docData, { merge: true });
+                } else {
+                    destBatch.set(destDocRef, docData);
+                }
+            }
+
+            batchDocIds.push(doc.id);
+            stats.documentsTransferred++;
+            if (progressBar) {
+                progressBar.increment();
+            }
+
+            logger.info('Transferred document', {
+                source: collectionPath,
+                dest: destCollectionPath,
+                sourceDocId: doc.id,
+                destDocId: destDocId,
+            });
+
+            if (config.includeSubcollections) {
+                const subcollections = await getSubcollections(doc.ref);
+
+                for (const subcollectionId of subcollections) {
+                    // Check exclude patterns
+                    if (matchesExcludePattern(subcollectionId, config.exclude)) {
+                        logger.info(`Skipping excluded subcollection: ${subcollectionId}`);
+                        continue;
+                    }
+
+                    const subcollectionPath = `${collectionPath}/${doc.id}/${subcollectionId}`;
+
+                    await transferCollection(
+                        { ...ctx, config: { ...config, limit: 0, where: [] } },
+                        subcollectionPath,
+                        depth + 1
+                    );
+                }
+            }
+        }
+
+        if (!config.dryRun && batch.length > 0) {
+            // Apply rate limiting before commit
+            if (rateLimiter) {
+                await rateLimiter.acquire(batchDocIds.length);
+            }
+
+            await withRetry(() => destBatch.commit(), {
+                retries: config.retries,
+                onRetry: (attempt, max, err, delay) => {
+                    logger.error(`Retry commit ${attempt}/${max}`, { error: err.message, delay });
+                },
+            });
+
+            // Save state after successful batch commit (for resume support)
+            if (state && batchDocIds.length > 0) {
+                for (const docId of batchDocIds) {
+                    markDocCompleted(state, collectionPath, docId);
+                }
+                state.stats = { ...stats };
+                saveTransferState(config.stateFile, state);
+            }
+        }
+    }
+}