@fazetitans/fscopy 1.1.3 → 1.2.1

package/src/state/index.ts

@@ -1,8 +1,193 @@
 import fs from 'node:fs';
-import type { Config, TransferState } from '../types.js';
+import type { Config, TransferState, Stats } from '../types.js';
 
 export const STATE_VERSION = 1;
 
+// =============================================================================
+// CompletedDocsCache - O(1) lookup using Set instead of Array.includes()
+// =============================================================================
+
+/**
+ * Cache for completed document IDs using Set for O(1) lookups.
+ * Wraps the TransferState.completedDocs (which uses arrays for JSON serialization).
+ */
+export class CompletedDocsCache {
+    private readonly cache: Map<string, Set<string>> = new Map();
+
+    constructor(completedDocs: Record<string, string[]> = {}) {
+        for (const [collection, docIds] of Object.entries(completedDocs)) {
+            this.cache.set(collection, new Set(docIds));
+        }
+    }
+
+    /**
+     * Check if a document is completed. O(1) lookup.
+     */
+    has(collectionPath: string, docId: string): boolean {
+        return this.cache.get(collectionPath)?.has(docId) ?? false;
+    }
+
+    /**
+     * Mark a document as completed.
+     */
+    add(collectionPath: string, docId: string): void {
+        let set = this.cache.get(collectionPath);
+        if (!set) {
+            set = new Set();
+            this.cache.set(collectionPath, set);
+        }
+        set.add(docId);
+    }
+
+    /**
+     * Mark multiple documents as completed.
+     */
+    addBatch(collectionPath: string, docIds: string[]): void {
+        let set = this.cache.get(collectionPath);
+        if (!set) {
+            set = new Set();
+            this.cache.set(collectionPath, set);
+        }
+        for (const docId of docIds) {
+            set.add(docId);
+        }
+    }
+
+    /**
+     * Convert back to Record<string, string[]> for JSON serialization.
+     */
+    toRecord(): Record<string, string[]> {
+        const result: Record<string, string[]> = {};
+        for (const [collection, set] of this.cache) {
+            result[collection] = Array.from(set);
+        }
+        return result;
+    }
+
+    /**
+     * Get total count of completed documents.
+     */
+    get totalCount(): number {
+        let count = 0;
+        for (const set of this.cache.values()) {
+            count += set.size;
+        }
+        return count;
+    }
+}
+
+// =============================================================================
+// StateSaver
+// =============================================================================
+
+export interface StateSaverOptions {
+    /** Save every N batches (default: 10) */
+    batchInterval?: number;
+    /** Save every N milliseconds (default: 5000) */
+    timeInterval?: number;
+}
+
+const DEFAULT_BATCH_INTERVAL = 10;
+const DEFAULT_TIME_INTERVAL = 5000;
+
+/**
+ * Throttled state saver with O(1) completed doc lookups.
+ * Uses CompletedDocsCache for efficient lookups during transfer.
+ * Saves state every N batches OR after X milliseconds, whichever comes first.
+ */
+export class StateSaver {
+    private lastSaveTime: number = Date.now();
+    private batchesSinceLastSave: number = 0;
+    private readonly batchInterval: number;
+    private readonly timeInterval: number;
+    private dirty: boolean = false;
+    private readonly cache: CompletedDocsCache;
+
+    constructor(
+        private readonly stateFile: string,
+        private readonly state: TransferState,
+        options: StateSaverOptions = {}
+    ) {
+        this.batchInterval = options.batchInterval ?? DEFAULT_BATCH_INTERVAL;
+        this.timeInterval = options.timeInterval ?? DEFAULT_TIME_INTERVAL;
+        this.cache = new CompletedDocsCache(state.completedDocs);
+    }
+
+    /**
+     * Check if a document is already completed. O(1) lookup.
+     */
+    isCompleted(collectionPath: string, docId: string): boolean {
+        return this.cache.has(collectionPath, docId);
+    }
+
+    /**
+     * Mark documents as completed and update stats.
+     * Saves to disk if thresholds are met.
+     */
+    markBatchCompleted(collectionPath: string, docIds: string[], stats: Stats): void {
+        this.cache.addBatch(collectionPath, docIds);
+        this.state.stats = { ...stats };
+        this.dirty = true;
+        this.batchesSinceLastSave++;
+
+        if (this.shouldSave()) {
+            this.save();
+        }
+    }
+
+    /**
+     * Check if we should save based on batch count or time elapsed.
+     */
+    private shouldSave(): boolean {
+        if (this.batchesSinceLastSave >= this.batchInterval) {
+            return true;
+        }
+
+        const elapsed = Date.now() - this.lastSaveTime;
+        if (elapsed >= this.timeInterval) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Sync cache to state and save to disk.
+     */
+    private save(): void {
+        this.state.completedDocs = this.cache.toRecord();
+        saveTransferState(this.stateFile, this.state);
+        this.lastSaveTime = Date.now();
+        this.batchesSinceLastSave = 0;
+        this.dirty = false;
+    }
+
+    /**
+     * Force save if there are unsaved changes.
+     * Call this before shutdown or on completion.
+     */
+    flush(): void {
+        if (this.dirty) {
+            this.save();
+        }
+    }
+
+    /**
+     * Get the underlying state object.
+     * Note: completedDocs may be stale until flush() is called.
+     */
+    getState(): TransferState {
+        return this.state;
+    }
+
+    /**
+     * Get total count of completed documents.
+     */
+    get completedCount(): number {
+        return this.cache.totalCount;
+    }
+}
+
 export function loadTransferState(stateFile: string): TransferState | null {
     try {
         if (!fs.existsSync(stateFile)) {
@@ -73,6 +258,8 @@ export function createInitialState(config: Config): TransferState {
             documentsTransferred: 0,
             documentsDeleted: 0,
             errors: 0,
+            conflicts: 0,
+            integrityErrors: 0,
         },
     };
 }

package/src/transfer/clear.ts

@@ -1,156 +1,214 @@
-import type { Firestore } from 'firebase-admin/firestore';
+import type { Firestore, QueryDocumentSnapshot } from 'firebase-admin/firestore';
 import type { Config } from '../types.js';
-import type { Logger } from '../utils/logger.js';
+import type { Output } from '../utils/output.js';
 import { withRetry } from '../utils/retry.js';
 import { matchesExcludePattern } from '../utils/patterns.js';
 import { getSubcollections, getDestCollectionPath } from './helpers.js';
 
-export async function clearCollection(
+async function clearDocSubcollections(
     db: Firestore,
+    doc: QueryDocumentSnapshot,
     collectionPath: string,
     config: Config,
-    logger: Logger,
-    includeSubcollections: boolean
+    output: Output
 ): Promise<number> {
     let deletedCount = 0;
-    const collectionRef = db.collection(collectionPath);
-    const snapshot = await collectionRef.get();
+    const subcollections = await getSubcollections(doc.ref);
+
+    for (const subId of subcollections) {
+        if (matchesExcludePattern(subId, config.exclude)) continue;
 
-    if (snapshot.empty) {
-        return 0;
+        const subPath = `${collectionPath}/${doc.id}/${subId}`;
+        deletedCount += await clearCollection(db, subPath, config, output, true);
     }
 
+    return deletedCount;
+}
+
+async function deleteBatch(
+    db: Firestore,
+    batch: QueryDocumentSnapshot[],
+    collectionPath: string,
+    config: Config,
+    output: Output
+): Promise<number> {
+    const writeBatch = db.batch();
+
+    for (const doc of batch) {
+        writeBatch.delete(doc.ref);
+    }
+
+    if (!config.dryRun) {
+        await withRetry(() => writeBatch.commit(), {
+            retries: config.retries,
+            onRetry: (attempt, max, err, delay) => {
+                output.logError(`Retry delete ${attempt}/${max} for ${collectionPath}`, {
+                    error: err.message,
+                    delay,
+                });
+            },
+        });
+    }
+
+    output.logInfo(`Deleted ${batch.length} documents from ${collectionPath}`);
+    return batch.length;
+}
+
+export async function clearCollection(
+    db: Firestore,
+    collectionPath: string,
+    config: Config,
+    output: Output,
+    includeSubcollections: boolean
+): Promise<number> {
+    const snapshot = await db.collection(collectionPath).get();
+    if (snapshot.empty) return 0;
+
+    let deletedCount = 0;
+
     // Delete subcollections first if enabled
     if (includeSubcollections) {
         for (const doc of snapshot.docs) {
-            const subcollections = await getSubcollections(doc.ref);
-            for (const subId of subcollections) {
-                // Check exclude patterns
-                if (matchesExcludePattern(subId, config.exclude)) {
-                    continue;
-                }
-                const subPath = `${collectionPath}/${doc.id}/${subId}`;
-                deletedCount += await clearCollection(db, subPath, config, logger, true);
-            }
+            deletedCount += await clearDocSubcollections(db, doc, collectionPath, config, output);
         }
     }
 
     // Delete documents in batches
-    const docs = snapshot.docs;
-    for (let i = 0; i < docs.length; i += config.batchSize) {
-        const batch = docs.slice(i, i + config.batchSize);
-        const writeBatch = db.batch();
-
-        for (const doc of batch) {
-            writeBatch.delete(doc.ref);
-            deletedCount++;
+    for (let i = 0; i < snapshot.docs.length; i += config.batchSize) {
+        const batch = snapshot.docs.slice(i, i + config.batchSize);
+        deletedCount += await deleteBatch(db, batch, collectionPath, config, output);
+    }
+
+    return deletedCount;
+}
+
+async function clearOrphanSubcollections(
+    destDb: Firestore,
+    doc: QueryDocumentSnapshot,
+    destCollectionPath: string,
+    config: Config,
+    output: Output
+): Promise<number> {
+    let deletedCount = 0;
+    const subcollections = await getSubcollections(doc.ref);
+
+    for (const subId of subcollections) {
+        if (matchesExcludePattern(subId, config.exclude)) continue;
+
+        const subPath = `${destCollectionPath}/${doc.id}/${subId}`;
+        deletedCount += await clearCollection(destDb, subPath, config, output, true);
+    }
+
+    return deletedCount;
+}
+
+async function deleteOrphanBatch(
+    destDb: Firestore,
+    batch: QueryDocumentSnapshot[],
+    destCollectionPath: string,
+    config: Config,
+    output: Output
+): Promise<number> {
+    let deletedCount = 0;
+    const writeBatch = destDb.batch();
+
+    for (const doc of batch) {
+        if (config.includeSubcollections) {
+            deletedCount += await clearOrphanSubcollections(
+                destDb,
+                doc,
+                destCollectionPath,
+                config,
+                output
+            );
         }
+        writeBatch.delete(doc.ref);
+        deletedCount++;
+    }
 
-        if (!config.dryRun) {
-            await withRetry(() => writeBatch.commit(), {
-                retries: config.retries,
-                onRetry: (attempt, max, err, delay) => {
-                    logger.error(`Retry delete ${attempt}/${max} for ${collectionPath}`, {
+    if (!config.dryRun) {
+        await withRetry(() => writeBatch.commit(), {
+            retries: config.retries,
+            onRetry: (attempt, max, err, delay) => {
+                output.logError(
+                    `Retry delete orphans ${attempt}/${max} for ${destCollectionPath}`,
+                    {
                         error: err.message,
                         delay,
-                    });
-                },
-            });
-        }
-
-        logger.info(`Deleted ${batch.length} documents from ${collectionPath}`);
+                    }
+                );
+            },
+        });
     }
 
+    output.logInfo(`Deleted ${batch.length} orphan documents from ${destCollectionPath}`);
     return deletedCount;
 }
 
-export async function deleteOrphanDocuments(
+async function processSubcollectionOrphans(
     sourceDb: Firestore,
     destDb: Firestore,
+    sourceSnapshot: FirebaseFirestore.QuerySnapshot,
     sourceCollectionPath: string,
     config: Config,
-    logger: Logger
+    output: Output
 ): Promise<number> {
     let deletedCount = 0;
 
-    // Get the destination path (may be renamed)
+    for (const sourceDoc of sourceSnapshot.docs) {
+        const sourceSubcollections = await getSubcollections(sourceDoc.ref);
+        for (const subId of sourceSubcollections) {
+            if (matchesExcludePattern(subId, config.exclude)) continue;
+
+            const subPath = `${sourceCollectionPath}/${sourceDoc.id}/${subId}`;
+            deletedCount += await deleteOrphanDocuments(sourceDb, destDb, subPath, config, output);
+        }
+    }
+
+    return deletedCount;
+}
+
+export async function deleteOrphanDocuments(
+    sourceDb: Firestore,
+    destDb: Firestore,
+    sourceCollectionPath: string,
+    config: Config,
+    output: Output
+): Promise<number> {
     const destCollectionPath = getDestCollectionPath(sourceCollectionPath, config.renameCollection);
 
-    // Get all document IDs from source (use select() to only fetch IDs, not data)
     const sourceSnapshot = await sourceDb.collection(sourceCollectionPath).select().get();
     const sourceIds = new Set(sourceSnapshot.docs.map((doc) => doc.id));
 
-    // Get all document IDs from destination (use select() to only fetch IDs, not data)
     const destSnapshot = await destDb.collection(destCollectionPath).select().get();
-
-    // Find orphan documents (in dest but not in source)
     const orphanDocs = destSnapshot.docs.filter((doc) => !sourceIds.has(doc.id));
 
-    if (orphanDocs.length === 0) {
-        return 0;
-    }
-
-    logger.info(`Found ${orphanDocs.length} orphan documents in ${destCollectionPath}`);
-
-    // Delete orphan documents in batches
-    for (let i = 0; i < orphanDocs.length; i += config.batchSize) {
-        const batch = orphanDocs.slice(i, i + config.batchSize);
-        const writeBatch = destDb.batch();
-
-        for (const doc of batch) {
-            // If subcollections are included, recursively delete orphans in subcollections first
-            if (config.includeSubcollections) {
-                const subcollections = await getSubcollections(doc.ref);
-                for (const subId of subcollections) {
-                    if (matchesExcludePattern(subId, config.exclude)) {
-                        continue;
-                    }
-                    const subPath = `${destCollectionPath}/${doc.id}/${subId}`;
-                    // For orphan parent docs, clear all subcollection data
-                    deletedCount += await clearCollection(destDb, subPath, config, logger, true);
-                }
-            }
-
-            writeBatch.delete(doc.ref);
-            deletedCount++;
-        }
+    let deletedCount = 0;
 
-        if (!config.dryRun) {
-            await withRetry(() => writeBatch.commit(), {
-                retries: config.retries,
-                onRetry: (attempt, max, err, delay) => {
-                    logger.error(
-                        `Retry delete orphans ${attempt}/${max} for ${destCollectionPath}`,
-                        {
-                            error: err.message,
-                            delay,
-                        }
-                    );
-                },
-            });
+    if (orphanDocs.length > 0) {
+        output.logInfo(`Found ${orphanDocs.length} orphan documents in ${destCollectionPath}`);
+
+        for (let i = 0; i < orphanDocs.length; i += config.batchSize) {
+            const batch = orphanDocs.slice(i, i + config.batchSize);
+            deletedCount += await deleteOrphanBatch(
+                destDb,
+                batch,
+                destCollectionPath,
+                config,
+                output
+            );
         }
-
-        logger.info(`Deleted ${batch.length} orphan documents from ${destCollectionPath}`);
     }
 
-    // Also check subcollections of existing documents for orphans
     if (config.includeSubcollections) {
-        for (const sourceDoc of sourceSnapshot.docs) {
-            const sourceSubcollections = await getSubcollections(sourceDoc.ref);
-            for (const subId of sourceSubcollections) {
-                if (matchesExcludePattern(subId, config.exclude)) {
-                    continue;
-                }
-                const subPath = `${sourceCollectionPath}/${sourceDoc.id}/${subId}`;
-                deletedCount += await deleteOrphanDocuments(
-                    sourceDb,
-                    destDb,
-                    subPath,
-                    config,
-                    logger
-                );
-            }
-        }
+        deletedCount += await processSubcollectionOrphans(
+            sourceDb,
+            destDb,
+            sourceSnapshot,
+            sourceCollectionPath,
+            config,
+            output
+        );
     }
 
     return deletedCount;

package/src/transfer/count.ts

@@ -1,4 +1,4 @@
-import type { Firestore } from 'firebase-admin/firestore';
+import type { Firestore, Query } from 'firebase-admin/firestore';
 import type { Config } from '../types.js';
 import { matchesExcludePattern } from '../utils/patterns.js';
 import { getSubcollections } from './helpers.js';
@@ -8,64 +8,114 @@ export interface CountProgress {
     onSubcollection?: (path: string) => void;
 }
 
-export async function countDocuments(
+function buildQueryWithFilters(
     sourceDb: Firestore,
     collectionPath: string,
     config: Config,
-    depth: number = 0,
-    progress?: CountProgress
-): Promise<number> {
-    let count = 0;
-
-    // Build query with where filters (only at root level)
-    let query: FirebaseFirestore.Query = sourceDb.collection(collectionPath);
+    depth: number
+): Query {
+    let query: Query = sourceDb.collection(collectionPath);
     if (depth === 0 && config.where.length > 0) {
         for (const filter of config.where) {
             query = query.where(filter.field, filter.operator, filter.value);
         }
     }
+    return query;
+}
 
-    // Use count() aggregation to avoid downloading all documents (much cheaper)
-    // But we need document refs for subcollections, so we'll need a different approach
-    if (config.includeSubcollections) {
-        // When including subcollections, we need to fetch docs to get their refs
-        // Use select() to only fetch document IDs, not the data (reduces bandwidth)
-        const snapshot = await query.select().get();
-        count += snapshot.size;
-
-        // Report progress for root collections
-        if (depth === 0 && progress?.onCollection) {
-            progress.onCollection(collectionPath, snapshot.size);
-        }
+async function countWithSubcollections(
+    sourceDb: Firestore,
+    query: Query,
+    collectionPath: string,
+    config: Config,
+    depth: number,
+    progress?: CountProgress
+): Promise<number> {
+    // Apply limit at root level only
+    if (depth === 0 && config.limit > 0) {
+        query = query.limit(config.limit);
+    }
 
-        for (const doc of snapshot.docs) {
-            const subcollections = await getSubcollections(doc.ref);
-            for (const subId of subcollections) {
-                const subPath = `${collectionPath}/${doc.id}/${subId}`;
+    const snapshot = await query.select().get();
+    let count = snapshot.size;
 
-                // Check exclude patterns
-                if (matchesExcludePattern(subId, config.exclude)) {
-                    continue;
-                }
+    if (depth === 0 && progress?.onCollection) {
+        progress.onCollection(collectionPath, snapshot.size);
+    }
 
-                // Report subcollection discovery
-                if (progress?.onSubcollection) {
-                    progress.onSubcollection(subPath);
-                }
+    for (const doc of snapshot.docs) {
+        count += await countSubcollectionsForDoc(
+            sourceDb,
+            doc,
+            collectionPath,
+            config,
+            depth,
+            progress
+        );
+    }
 
-                count += await countDocuments(sourceDb, subPath, config, depth + 1, progress);
-            }
-        }
-    } else {
-        // No subcollections: use count() aggregation (1 read instead of N)
-        const countSnapshot = await query.count().get();
-        count = countSnapshot.data().count;
-
-        // Report progress for root collections
-        if (depth === 0 && progress?.onCollection) {
-            progress.onCollection(collectionPath, count);
+    return count;
+}
+
+async function countSubcollectionsForDoc(
+    sourceDb: Firestore,
+    doc: FirebaseFirestore.QueryDocumentSnapshot,
+    collectionPath: string,
+    config: Config,
+    depth: number,
+    progress?: CountProgress
+): Promise<number> {
+    let count = 0;
+    const subcollections = await getSubcollections(doc.ref);
+
+    for (const subId of subcollections) {
+        if (matchesExcludePattern(subId, config.exclude)) continue;
+
+        const subPath = `${collectionPath}/${doc.id}/${subId}`;
+        if (progress?.onSubcollection) {
+            progress.onSubcollection(subPath);
         }
+
+        count += await countDocuments(sourceDb, subPath, config, depth + 1, progress);
+    }
+
+    return count;
+}
+
+async function countWithoutSubcollections(
+    query: Query,
+    collectionPath: string,
+    config: Config,
+    depth: number,
+    progress?: CountProgress
+): Promise<number> {
+    const countSnapshot = await query.count().get();
+    let count = countSnapshot.data().count;
+
+    // Apply limit at root level only
+    if (depth === 0 && config.limit > 0) {
+        count = Math.min(count, config.limit);
+    }
+
+    if (depth === 0 && progress?.onCollection) {
+        progress.onCollection(collectionPath, count);
     }
 
     return count;
 }
+
+export async function countDocuments(
+    sourceDb: Firestore,
+    collectionPath: string,
+    config: Config,
+    depth: number = 0,
+    progress?: CountProgress
+): Promise<number> {
+    const query = buildQueryWithFilters(sourceDb, collectionPath, config, depth);
+
+    if (config.includeSubcollections) {
+        return countWithSubcollections(sourceDb, query, collectionPath, config, depth, progress);
+    }
+
+    return countWithoutSubcollections(query, collectionPath, config, depth, progress);
+}