@fazetitans/fscopy 1.3.1 → 1.5.0

package/src/transfer/count.ts

@@ -1,26 +1,13 @@
-import type { Firestore, Query } from 'firebase-admin/firestore';
+import type { Firestore, Query, QueryDocumentSnapshot } from 'firebase-admin/firestore';
 import type { Config } from '../types.js';
 import { matchesExcludePattern } from '../utils/patterns.js';
-import { getSubcollections } from './helpers.js';
+import { getSubcollections, buildQueryWithFilters } from './helpers.js';
+import { CLEAR_PAGE_SIZE } from '../constants.js';
 
 export interface CountProgress {
     onCollection?: (path: string, count: number) => void;
     onSubcollection?: (path: string) => void;
-}
-
-function buildQueryWithFilters(
-    sourceDb: Firestore,
-    collectionPath: string,
-    config: Config,
-    depth: number
-): Query {
-    let query: Query = sourceDb.collection(collectionPath);
-    if (depth === 0 && config.where.length > 0) {
-        for (const filter of config.where) {
-            query = query.where(filter.field, filter.operator, filter.value);
-        }
-    }
-    return query;
+    onSubcollectionExcluded?: (name: string) => void;
 }
 
 async function countWithSubcollections(
@@ -31,30 +18,53 @@ async function countWithSubcollections(
     depth: number,
     progress?: CountProgress
 ): Promise<number> {
-    // Apply limit at root level only
-    if (depth === 0 && config.limit > 0) {
-        query = query.limit(config.limit);
-    }
+    const userLimit = depth === 0 && config.limit > 0 ? config.limit : 0;
+    let rootCount = 0;
+    let subCount = 0;
+    let lastDoc: QueryDocumentSnapshot | undefined;
+
+    while (true) {
+        let pageSize = CLEAR_PAGE_SIZE;
+        if (userLimit > 0) {
+            const remaining = userLimit - rootCount;
+            if (remaining <= 0) break;
+            pageSize = Math.min(pageSize, remaining);
+        }
 
-    const snapshot = await query.select().get();
-    let count = snapshot.size;
+        let pageQuery = query.select().limit(pageSize);
+        if (lastDoc) {
+            pageQuery = pageQuery.startAfter(lastDoc);
+        }
 
-    if (depth === 0 && progress?.onCollection) {
-        progress.onCollection(collectionPath, snapshot.size);
+        const snapshot = await pageQuery.get();
+        if (snapshot.empty) break;
+
+        rootCount += snapshot.size;
+
+        if (depth === 0 && progress?.onCollection) {
+            progress.onCollection(collectionPath, rootCount);
+        }
+
+        for (const doc of snapshot.docs) {
+            subCount += await countSubcollectionsForDoc(
+                sourceDb,
+                doc,
+                collectionPath,
+                config,
+                depth,
+                progress
+            );
+        }
+
+        lastDoc = snapshot.docs[snapshot.docs.length - 1];
+        if (snapshot.size < pageSize) break;
     }
 
-    for (const doc of snapshot.docs) {
-        count += await countSubcollectionsForDoc(
-            sourceDb,
-            doc,
-            collectionPath,
-            config,
-            depth,
-            progress
-        );
+    if (rootCount === 0 && depth === 0 && progress?.onCollection) {
+        progress.onCollection(collectionPath, 0);
     }
 
-    return count;
+    return rootCount + subCount;
 }
 
 async function countSubcollectionsForDoc(
@@ -65,11 +75,19 @@ async function countSubcollectionsForDoc(
     depth: number,
     progress?: CountProgress
 ): Promise<number> {
+    // Respect maxDepth to match transfer behavior
+    if (config.maxDepth > 0 && depth >= config.maxDepth) return 0;
+
     let count = 0;
     const subcollections = await getSubcollections(doc.ref);
 
     for (const subId of subcollections) {
-        if (matchesExcludePattern(subId, config.exclude)) continue;
+        if (matchesExcludePattern(subId, config.exclude)) {
+            if (progress?.onSubcollectionExcluded) {
+                progress.onSubcollectionExcluded(subId);
+            }
+            continue;
+        }
 
         const subPath = `${collectionPath}/${doc.id}/${subId}`;
         if (progress?.onSubcollection) {

package/src/transfer/helpers.ts

@@ -1,4 +1,6 @@
-import type { DocumentReference } from 'firebase-admin/firestore';
+import type { DocumentReference, Firestore, Query } from 'firebase-admin/firestore';
+import type { Config } from '../types.js';
+import { matchesExcludePattern } from '../utils/patterns.js';
 
 export async function getSubcollections(docRef: DocumentReference): Promise<string[]> {
     const collections = await docRef.listCollections();
@@ -35,3 +37,36 @@ export function getDestDocId(
     }
     return destId;
 }
+
+/**
+ * Get non-excluded subcollection IDs for a document.
+ * Filters out subcollections matching exclude patterns.
+ */
+export async function getFilteredSubcollections(
+    docRef: DocumentReference,
+    exclude: string[]
+): Promise<string[]> {
+    const subcollections = await getSubcollections(docRef);
+    return subcollections.filter((id) => !matchesExcludePattern(id, exclude));
+}
+
+/**
+ * Build a Firestore query with where filters applied.
+ * Filters are only applied at root level (depth === 0).
+ */
+export function buildQueryWithFilters(
+    sourceDb: Firestore,
+    collectionPath: string,
+    config: Config,
+    depth: number
+): Query {
+    let query: Query = sourceDb.collection(collectionPath);
+
+    if (depth === 0 && config.where.length > 0) {
+        for (const filter of config.where) {
+            query = query.where(filter.field, filter.operator, filter.value);
+        }
+    }
+
+    return query;
+}

package/src/transfer/index.ts

@@ -1,4 +1,10 @@
-export { getSubcollections, getDestCollectionPath, getDestDocId } from './helpers.js';
+export {
+    getSubcollections,
+    getDestCollectionPath,
+    getDestDocId,
+    getFilteredSubcollections,
+    buildQueryWithFilters,
+} from './helpers.js';
 export { processInParallel, type ParallelResult } from './parallel.js';
 export { countDocuments, type CountProgress } from './count.js';
 export { clearCollection, deleteOrphanDocuments, type DeleteOrphansProgress } from './clear.js';

package/src/transfer/transfer.ts

@@ -1,14 +1,23 @@
-import type { Firestore, WriteBatch, Query, QueryDocumentSnapshot } from 'firebase-admin/firestore';
+import {
+    FieldPath,
+    type Firestore,
+    type WriteBatch,
+    type QueryDocumentSnapshot,
+} from 'firebase-admin/firestore';
 import type { Config, Stats, TransformFunction, ConflictInfo } from '../types.js';
 import type { Output } from '../utils/output.js';
 import type { RateLimiter } from '../utils/rate-limiter.js';
 import type { ProgressBarWrapper } from '../utils/progress.js';
 import type { StateSaver } from '../state/index.js';
 import { withRetry } from '../utils/retry.js';
-import { matchesExcludePattern } from '../utils/patterns.js';
 import { estimateDocumentSize, formatBytes, FIRESTORE_MAX_DOC_SIZE } from '../utils/doc-size.js';
 import { hashDocumentData, compareHashes } from '../utils/integrity.js';
-import { getSubcollections, getDestCollectionPath, getDestDocId } from './helpers.js';
+import {
+    getDestCollectionPath,
+    getDestDocId,
+    getFilteredSubcollections,
+    buildQueryWithFilters,
+} from './helpers.js';
 
 export interface TransferContext {
     sourceDb: Firestore;
@@ -21,6 +30,7 @@ export interface TransferContext {
     stateSaver: StateSaver | null;
     rateLimiter: RateLimiter | null;
     conflictList: ConflictInfo[];
+    maxDepthWarningsShown: Set<string>;
 }
 
 interface DocProcessResult {
@@ -29,99 +39,6 @@ interface DocProcessResult {
     markCompleted: boolean;
 }
 
-// Map of destDocId -> updateTime (as ISO string for comparison)
-type UpdateTimeMap = Map<string, string | null>;
-
-/**
- * Capture updateTime of destination documents before processing.
- * Returns a map of docId -> updateTime (ISO string, or null if doc doesn't exist).
- */
-async function captureDestUpdateTimes(
-    destDb: Firestore,
-    destCollectionPath: string,
-    destDocIds: string[]
-): Promise<UpdateTimeMap> {
-    const updateTimes: UpdateTimeMap = new Map();
-
-    // Batch get dest docs to get their updateTime
-    const docRefs = destDocIds.map((id) => destDb.collection(destCollectionPath).doc(id));
-    const docs = await destDb.getAll(...docRefs);
-
-    for (let i = 0; i < docs.length; i++) {
-        const doc = docs[i];
-        const docId = destDocIds[i];
-        if (doc.exists) {
-            const updateTime = doc.updateTime;
-            updateTimes.set(docId, updateTime ? updateTime.toDate().toISOString() : null);
-        } else {
-            updateTimes.set(docId, null);
-        }
-    }
-
-    return updateTimes;
-}
-
-/**
- * Check for conflicts by comparing current updateTimes with captured ones.
- * Returns array of docIds that have conflicts.
- */
-async function checkForConflicts(
-    destDb: Firestore,
-    destCollectionPath: string,
-    destDocIds: string[],
-    capturedTimes: UpdateTimeMap
-): Promise<string[]> {
-    const conflicts: string[] = [];
-
-    const docRefs = destDocIds.map((id) => destDb.collection(destCollectionPath).doc(id));
-    const docs = await destDb.getAll(...docRefs);
-
-    for (let i = 0; i < docs.length; i++) {
-        const doc = docs[i];
-        const docId = destDocIds[i];
-        const capturedTime = capturedTimes.get(docId);
-
-        const currentTime =
-            doc.exists && doc.updateTime ? doc.updateTime.toDate().toISOString() : null;
-
-        // Conflict conditions:
-        // 1. Doc didn't exist before but now exists (created by someone else)
-        // 2. Doc was modified (updateTime changed)
-        // 3. Doc was deleted during transfer (existed before, doesn't now)
-        const isConflict =
-            (doc.exists && capturedTime === null) ||
-            (doc.exists && currentTime !== capturedTime) ||
-            (!doc.exists && capturedTime !== null);
-
-        if (isConflict) {
-            conflicts.push(docId);
-        }
-    }
-
-    return conflicts;
-}
-
-function buildTransferQuery(
-    sourceDb: Firestore,
-    collectionPath: string,
-    config: Config,
-    depth: number
-): Query {
-    let query: Query = sourceDb.collection(collectionPath);
-
-    if (depth === 0 && config.where.length > 0) {
-        for (const filter of config.where) {
-            query = query.where(filter.field, filter.operator, filter.value);
-        }
-    }
-
-    if (config.limit > 0 && depth === 0) {
-        query = query.limit(config.limit);
-    }
-
-    return query;
-}
-
 function applyTransform(
     docData: Record<string, unknown>,
     doc: QueryDocumentSnapshot,
@@ -152,6 +69,7 @@ function applyTransform(
             collection: collectionPath,
             error: errMsg,
         });
+        output.warn(`⚠️  Transform error: ${collectionPath}/${doc.id} skipped (${errMsg})`);
         stats.errors++;
         return { success: false, data: null, markCompleted: false };
     }
@@ -186,9 +104,6 @@ function checkDocumentSize(
     );
 }
 
-// Track which collections have already shown the max-depth warning (to avoid spam)
-const maxDepthWarningsShown = new Set<string>();
-
 async function processSubcollections(
     ctx: TransferContext,
     doc: QueryDocumentSnapshot,
@@ -201,8 +116,8 @@ async function processSubcollections(
     if (config.maxDepth > 0 && depth >= config.maxDepth) {
         // Show console warning only once per root collection
         const rootCollection = collectionPath.split('/')[0];
-        if (!maxDepthWarningsShown.has(rootCollection)) {
-            maxDepthWarningsShown.add(rootCollection);
+        if (!ctx.maxDepthWarningsShown.has(rootCollection)) {
+            ctx.maxDepthWarningsShown.add(rootCollection);
             output.warn(
                 `⚠️  Subcollections in ${rootCollection} beyond depth ${config.maxDepth} will be skipped`
             );
@@ -215,15 +130,21 @@ async function processSubcollections(
         return;
     }
 
-    const subcollections = await getSubcollections(doc.ref);
+    const subcollections = await getFilteredSubcollections(doc.ref, config.exclude);
 
     for (const subcollectionId of subcollections) {
-        if (matchesExcludePattern(subcollectionId, config.exclude)) {
-            output.logInfo(`Skipping excluded subcollection: ${subcollectionId}`);
-            continue;
+        const subcollectionPath = `${collectionPath}/${doc.id}/${subcollectionId}`;
+
+        // Count subcollection docs with .count() aggregation (1 read instead of N)
+        // and dynamically adjust the progress bar total
+        if (ctx.progressBar.isActive) {
+            const countSnap = await ctx.sourceDb.collection(subcollectionPath).count().get();
+            const subCount = countSnap.data().count;
+            if (subCount > 0) {
+                ctx.progressBar.addToTotal(subCount);
+            }
         }
 
-        const subcollectionPath = `${collectionPath}/${doc.id}/${subcollectionId}`;
         const subCtx = { ...ctx, config: { ...config, limit: 0, where: [] } };
         await transferCollection(subCtx, subcollectionPath, depth + 1);
     }
@@ -244,7 +165,18 @@ function processDocument(
     }
 
     const destDocId = getDestDocId(doc.id, config.idPrefix, config.idSuffix);
-    let docData = doc.data() as Record<string, unknown>;
+    let docData: Record<string, unknown>;
+    try {
+        docData = doc.data() as Record<string, unknown>;
+    } catch (error) {
+        const errMsg = error instanceof Error ? error.message : String(error);
+        output.logError(`Failed to read document data for ${doc.id}`, {
+            collection: collectionPath,
+            error: errMsg,
+        });
+        stats.errors++;
+        return { skip: true, markCompleted: false };
+    }
 
     // Apply transform if provided
     if (transformFn) {
@@ -295,12 +227,31 @@ async function commitBatchWithRetry(
         await rateLimiter.acquire(batchDocIds.length);
     }
 
-    await withRetry(() => destBatch.commit(), {
-        retries: config.retries,
-        onRetry: (attempt, max, err, delay) => {
-            output.logError(`Retry commit ${attempt}/${max}`, { error: err.message, delay });
-        },
-    });
+    try {
+        await withRetry(() => destBatch.commit(), {
+            retries: config.retries,
+            onRetry: (attempt, max, err, delay) => {
+                output.logError(`Retry commit ${attempt}/${max}`, { error: err.message, delay });
+            },
+        });
+    } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        stats.errors += batchDocIds.length;
+        output.logError(
+            `Batch commit failed for ${batchDocIds.length} documents after ${config.retries} retries`,
+            {
+                collection: collectionPath,
+                error: err.message,
+                docIds: batchDocIds.slice(0, 10),
+            }
+        );
+        output.warn(
+            `⚠️  Batch commit failed: ${batchDocIds.length} documents in ${collectionPath} were NOT written (${err.message})`
+        );
+        // Re-decrement documentsTransferred since they weren't actually committed
+        stats.documentsTransferred -= batchDocIds.length;
+        return;
+    }
 
     if (stateSaver && batchDocIds.length > 0) {
         stateSaver.markBatchCompleted(collectionPath, batchDocIds, stats);
@@ -365,9 +316,40 @@ async function verifyBatchIntegrity(
     preparedDocs: PreparedDoc[],
     destDb: Firestore,
     destCollectionPath: string,
+    merge: boolean,
     stats: Stats,
     output: Output
 ): Promise<void> {
+    if (!merge) {
+        // Non-merge mode: data written is exactly what we sent, no re-fetch needed.
+        // The source hash was computed from the same data we wrote, so they must match.
+        // We only need to verify the docs exist (spot-check a single doc for commit success).
+        const sampleRef = destDb.collection(destCollectionPath).doc(preparedDocs[0].destDocId);
+        const sampleDoc = await sampleRef.get();
+        if (!sampleDoc.exists) {
+            // Commit may have silently failed — verify all
+            const docRefs = preparedDocs.map((p) =>
+                destDb.collection(destCollectionPath).doc(p.destDocId)
+            );
+            const destDocs = await destDb.getAll(...docRefs);
+            for (let i = 0; i < destDocs.length; i++) {
+                if (!destDocs[i].exists) {
+                    stats.integrityErrors++;
+                    output.warn(
+                        `⚠️  Integrity error: ${destCollectionPath}/${preparedDocs[i].destDocId} not found after write`
+                    );
+                    output.logError('Integrity verification failed', {
+                        collection: destCollectionPath,
+                        docId: preparedDocs[i].destDocId,
+                        reason: 'document_not_found',
+                    });
+                }
+            }
+        }
+        return;
+    }
+
+    // Merge mode: re-fetch and compare hashes (merged result may differ from source)
     const docRefs = preparedDocs.map((p) => destDb.collection(destCollectionPath).doc(p.destDocId));
     const destDocs = await destDb.getAll(...docRefs);
 
@@ -450,7 +432,14 @@ async function commitPreparedDocs(
 
         // Verify integrity after commit if enabled
         if (config.verifyIntegrity) {
-            await verifyBatchIntegrity(preparedDocs, destDb, destCollectionPath, stats, output);
+            await verifyBatchIntegrity(
+                preparedDocs,
+                destDb,
+                destCollectionPath,
+                config.merge,
+                stats,
+                output
+            );
         }
     }
 
@@ -479,50 +468,50 @@ async function processBatch(
         return [];
     }
 
-    // Step 2: If conflict detection is enabled, capture dest updateTimes and check for conflicts
-    let docsToWrite = preparedDocs;
+    // Step 2: If conflict detection is enabled, check for existing docs in destination
+    // Uses chunked 'in' queries with .select() to minimize reads:
+    // - Firestore 'in' operator supports max 30 values per query
+    // - .select() avoids transferring field data (saves bandwidth)
+    // - Only existing docs cost reads; non-existent docs are free (unlike getAll)
     if (config.detectConflicts && !config.dryRun) {
         const destDocIds = preparedDocs.map((p) => p.destDocId);
-        const capturedTimes = await captureDestUpdateTimes(destDb, destCollectionPath, destDocIds);
-
-        // Check for conflicts
-        const conflictingIds = await checkForConflicts(
-            destDb,
-            destCollectionPath,
-            destDocIds,
-            capturedTimes
-        );
-
-        if (conflictingIds.length > 0) {
-            const conflictSet = new Set(conflictingIds);
-
-            // Filter out conflicting docs
-            docsToWrite = preparedDocs.filter((p) => !conflictSet.has(p.destDocId));
+        const existingIds = new Set<string>();
+        const FIRESTORE_IN_LIMIT = 30;
+
+        for (let i = 0; i < destDocIds.length; i += FIRESTORE_IN_LIMIT) {
+            const chunk = destDocIds.slice(i, i + FIRESTORE_IN_LIMIT);
+            const snapshot = await destDb
+                .collection(destCollectionPath)
+                .where(FieldPath.documentId(), 'in', chunk)
+                .select()
+                .get();
+            for (const doc of snapshot.docs) {
+                existingIds.add(doc.id);
+            }
+        }
 
-            // Record conflicts
-            for (const prepared of preparedDocs) {
-                if (conflictSet.has(prepared.destDocId)) {
-                    stats.conflicts++;
-                    conflictList.push({
-                        collection: destCollectionPath,
-                        docId: prepared.destDocId,
-                        reason: 'Document was modified during transfer',
-                    });
-                    output.warn(
-                        `⚠️  Conflict detected: ${destCollectionPath}/${prepared.destDocId} was modified during transfer`
-                    );
-                    output.logError('Conflict detected', {
-                        collection: destCollectionPath,
-                        docId: prepared.destDocId,
-                        reason: 'modified_during_transfer',
-                    });
-                }
+        if (existingIds.size > 0) {
+            for (const docId of existingIds) {
+                stats.conflicts++;
+                conflictList.push({
+                    collection: destCollectionPath,
+                    docId,
+                    reason: 'Document already exists in destination',
+                });
+                output.logError('Conflict detected', {
+                    collection: destCollectionPath,
+                    docId,
+                    reason: 'document_exists_in_destination',
+                });
             }
+            output.warn(
+                `⚠️  ${existingIds.size} document(s) already exist in ${destCollectionPath} and will be overwritten`
+            );
         }
     }
 
-    // Step 3: Commit non-conflicting docs
-    return commitPreparedDocs(docsToWrite, ctx, collectionPath, destCollectionPath, depth);
+    // Step 3: Commit docs
+    return commitPreparedDocs(preparedDocs, ctx, collectionPath, destCollectionPath, depth);
 }
 
 export async function transferCollection(
@@ -533,25 +522,50 @@ export async function transferCollection(
     const { sourceDb, config, stats, output } = ctx;
     const destCollectionPath = getDestCollectionPath(collectionPath, config.renameCollection);
 
-    const query = buildTransferQuery(sourceDb, collectionPath, config, depth);
+    const baseQuery = buildQueryWithFilters(sourceDb, collectionPath, config, depth);
+    const userLimit = config.limit > 0 && depth === 0 ? config.limit : 0;
 
-    const snapshot = await withRetry(() => query.get(), {
-        retries: config.retries,
-        onRetry: (attempt, max, err, delay) => {
-            output.logError(`Retry ${attempt}/${max} for ${collectionPath}`, {
-                error: err.message,
-                delay,
-            });
-        },
-    });
+    let totalProcessed = 0;
+    let lastDoc: QueryDocumentSnapshot | undefined;
+
+    while (true) {
+        // Calculate page size respecting user limit
+        let pageSize = config.batchSize;
+        if (userLimit > 0) {
+            const remaining = userLimit - totalProcessed;
+            if (remaining <= 0) break;
+            pageSize = Math.min(pageSize, remaining);
+        }
+
+        // Build paginated query
+        let pageQuery = baseQuery.limit(pageSize);
+        if (lastDoc) {
+            pageQuery = pageQuery.startAfter(lastDoc);
+        }
+
+        const snapshot = await withRetry(() => pageQuery.get(), {
+            retries: config.retries,
+            onRetry: (attempt, max, err, delay) => {
+                output.logError(`Retry ${attempt}/${max} for ${collectionPath}`, {
+                    error: err.message,
+                    delay,
+                });
+            },
+        });
+
+        if (snapshot.empty) break;
+
+        if (totalProcessed === 0) {
+            stats.collectionsProcessed++;
+            output.logInfo(`Processing collection: ${collectionPath}`);
+        }
 
-    if (snapshot.empty) return;
+        await processBatch(snapshot.docs, ctx, collectionPath, destCollectionPath, depth);
 
-    stats.collectionsProcessed++;
-    output.logInfo(`Processing collection: ${collectionPath}`, { documents: snapshot.size });
+        totalProcessed += snapshot.docs.length;
+        lastDoc = snapshot.docs[snapshot.docs.length - 1];
 
-    for (let i = 0; i < snapshot.docs.length; i += config.batchSize) {
-        const batch = snapshot.docs.slice(i, i + config.batchSize);
-        await processBatch(batch, ctx, collectionPath, destCollectionPath, depth);
+        // Fewer docs than requested means we've reached the end
+        if (snapshot.docs.length < pageSize) break;
     }
 }

package/src/transform/loader.ts

@@ -2,9 +2,19 @@ import fs from 'node:fs';
 import path from 'node:path';
 import type { TransformFunction } from '../types.js';
 
+const ALLOWED_EXTENSIONS = new Set(['.ts', '.js', '.mjs', '.mts']);
+
 export async function loadTransformFunction(transformPath: string): Promise<TransformFunction> {
     const absolutePath = path.resolve(transformPath);
 
+    // Validate file extension
+    const ext = path.extname(absolutePath).toLowerCase();
+    if (!ALLOWED_EXTENSIONS.has(ext)) {
+        throw new Error(
+            `Transform file must be a JavaScript or TypeScript file (${[...ALLOWED_EXTENSIONS].join(', ')}). Got: "${ext || '(no extension)'}"`
+        );
+    }
+
     if (!fs.existsSync(absolutePath)) {
         throw new Error(`Transform file not found: ${absolutePath}`);
     }
@@ -26,6 +36,8 @@ export async function loadTransformFunction(transformPath: string): Promise<Tran
         if ((error as Error).message.includes('Transform file')) {
             throw error;
         }
-        throw new Error(`Failed to load transform file: ${(error as Error).message}`);
+        throw new Error(`Failed to load transform file: ${(error as Error).message}`, {
+            cause: error,
+        });
     }
 }