@fazetitans/fscopy 1.1.1 → 1.1.2

package/src/types.ts

@@ -0,0 +1,101 @@
+// =============================================================================
+// Shared Types
+// =============================================================================
+
+export interface WhereFilter {
+    field: string;
+    operator: FirebaseFirestore.WhereFilterOp;
+    value: string | number | boolean;
+}
+
+export interface Config {
+    collections: string[];
+    includeSubcollections: boolean;
+    dryRun: boolean;
+    batchSize: number;
+    limit: number;
+    sourceProject: string | null;
+    destProject: string | null;
+    retries: number;
+    where: WhereFilter[];
+    exclude: string[];
+    merge: boolean;
+    parallel: number;
+    clear: boolean;
+    deleteMissing: boolean;
+    transform: string | null;
+    renameCollection: Record<string, string>;
+    idPrefix: string | null;
+    idSuffix: string | null;
+    webhook: string | null;
+    resume: boolean;
+    stateFile: string;
+    verify: boolean;
+    rateLimit: number;
+    skipOversized: boolean;
+    json: boolean;
+}
+
+export interface Stats {
+    collectionsProcessed: number;
+    documentsTransferred: number;
+    documentsDeleted: number;
+    errors: number;
+}
+
+export interface TransferState {
+    version: number;
+    sourceProject: string;
+    destProject: string;
+    collections: string[];
+    startedAt: string;
+    updatedAt: string;
+    completedDocs: Record<string, string[]>; // collectionPath -> array of doc IDs
+    stats: Stats;
+}
+
+export interface LogEntry {
+    timestamp: string;
+    level: string;
+    message: string;
+    [key: string]: unknown;
+}
+
+export type TransformFunction = (
+    doc: Record<string, unknown>,
+    meta: { id: string; path: string }
+) => Record<string, unknown> | null;
+
+export interface CliArgs {
+    init?: string;
+    config?: string;
+    collections?: string[];
+    includeSubcollections?: boolean;
+    dryRun?: boolean;
+    batchSize?: number;
+    limit?: number;
+    sourceProject?: string;
+    destProject?: string;
+    yes: boolean;
+    log?: string;
+    retries: number;
+    quiet: boolean;
+    where?: string[];
+    exclude?: string[];
+    merge?: boolean;
+    parallel?: number;
+    clear?: boolean;
+    deleteMissing?: boolean;
+    interactive?: boolean;
+    transform?: string;
+    renameCollection?: string[];
+    idPrefix?: string;
+    idSuffix?: string;
+    webhook?: string;
+    resume?: boolean;
+    stateFile?: string;
+    verify?: boolean;
+    rateLimit?: number;
+    skipOversized?: boolean;
+    json?: boolean;
+}

package/src/utils/credentials.ts

@@ -0,0 +1,32 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+export function checkCredentialsExist(): { exists: boolean; path: string } {
+    // Check for explicit credentials file
+    if (process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+        const credPath = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+        return { exists: fs.existsSync(credPath), path: credPath };
+    }
+
+    // Check for Application Default Credentials
+    const adcPath = path.join(
+        os.homedir(),
+        '.config',
+        'gcloud',
+        'application_default_credentials.json'
+    );
+    return { exists: fs.existsSync(adcPath), path: adcPath };
+}
+
+export function ensureCredentials(): void {
+    const { exists, path: credPath } = checkCredentialsExist();
+
+    if (!exists) {
+        console.error('\n❌ Google Cloud credentials not found.');
+        console.error(`   Expected at: ${credPath}\n`);
+        console.error('   Run this command to authenticate:');
+        console.error('   gcloud auth application-default login\n');
+        process.exit(1);
+    }
+}

package/src/utils/doc-size.ts

@@ -0,0 +1,129 @@
+/**
+ * Firestore maximum document size in bytes (1 MiB)
+ */
+export const FIRESTORE_MAX_DOC_SIZE = 1024 * 1024;
+
+/**
+ * Estimate the size of a Firestore document in bytes.
+ * This is an approximation - actual Firestore storage may differ slightly.
+ *
+ * Rules:
+ * - Strings: UTF-8 encoded length + 1
+ * - Numbers: 8 bytes
+ * - Booleans: 1 byte
+ * - Null: 1 byte
+ * - Timestamps: 8 bytes
+ * - Arrays: sum of element sizes
+ * - Maps: sum of key + value sizes
+ * - GeoPoints: 16 bytes
+ * - References: document path length + 1
+ */
+export function estimateDocumentSize(
+    data: Record<string, unknown>,
+    docPath?: string
+): number {
+    let size = 0;
+
+    // Document name (path) contributes to size
+    if (docPath) {
+        size += docPath.length + 1;
+    }
+
+    size += estimateValueSize(data);
+
+    return size;
+}
+
+function estimateValueSize(value: unknown): number {
+    if (value === null || value === undefined) {
+        return 1;
+    }
+
+    if (typeof value === 'boolean') {
+        return 1;
+    }
+
+    if (typeof value === 'number') {
+        return 8;
+    }
+
+    if (typeof value === 'string') {
+        // UTF-8 encoded length
+        return Buffer.byteLength(value, 'utf8') + 1;
+    }
+
+    if (value instanceof Date) {
+        return 8;
+    }
+
+    // Firestore Timestamp
+    if (
+        value &&
+        typeof value === 'object' &&
+        '_seconds' in value &&
+        '_nanoseconds' in value
+    ) {
+        return 8;
+    }
+
+    // GeoPoint
+    if (
+        value &&
+        typeof value === 'object' &&
+        '_latitude' in value &&
+        '_longitude' in value
+    ) {
+        return 16;
+    }
+
+    // DocumentReference
+    if (
+        value &&
+        typeof value === 'object' &&
+        '_path' in value &&
+        typeof (value as { _path: unknown })._path === 'object'
+    ) {
+        const pathObj = (value as { _path: { segments?: string[] } })._path;
+        if (pathObj.segments) {
+            return pathObj.segments.join('/').length + 1;
+        }
+        return 16; // Approximate
+    }
+
+    // Array
+    if (Array.isArray(value)) {
+        let size = 0;
+        for (const item of value) {
+            size += estimateValueSize(item);
+        }
+        return size;
+    }
+
+    // Map/Object
+    if (typeof value === 'object') {
+        let size = 0;
+        for (const [key, val] of Object.entries(value)) {
+            // Key size (field name)
+            size += key.length + 1;
+            // Value size
+            size += estimateValueSize(val);
+        }
+        return size;
+    }
+
+    // Unknown type - estimate as small value
+    return 8;
+}
+
+/**
+ * Format byte size to human-readable string
+ */
+export function formatBytes(bytes: number): string {
+    if (bytes < 1024) {
+        return `${bytes} B`;
+    }
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    }
+    return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+}

package/src/utils/errors.ts

@@ -0,0 +1,157 @@
+export interface FirebaseErrorInfo {
+    message: string;
+    suggestion?: string;
+}
+
+const errorMap: Record<string, FirebaseErrorInfo> = {
+    // Authentication errors
+    'auth/invalid-credential': {
+        message: 'Invalid credentials',
+        suggestion: 'Run "gcloud auth application-default login" to authenticate',
+    },
+    'app/invalid-credential': {
+        message: 'Invalid application credentials',
+        suggestion: 'Run "gcloud auth application-default login" to authenticate',
+    },
+
+    // Permission errors
+    'permission-denied': {
+        message: 'Permission denied',
+        suggestion: 'Ensure you have Firestore read/write access on this project',
+    },
+    'PERMISSION_DENIED': {
+        message: 'Permission denied',
+        suggestion: 'Ensure you have Firestore read/write access on this project',
+    },
+
+    // Network errors
+    unavailable: {
+        message: 'Service unavailable',
+        suggestion: 'Check your internet connection and try again',
+    },
+    UNAVAILABLE: {
+        message: 'Service unavailable',
+        suggestion: 'Check your internet connection and try again',
+    },
+
+    // Not found errors
+    'not-found': {
+        message: 'Resource not found',
+        suggestion: 'Verify the project ID and collection path are correct',
+    },
+    NOT_FOUND: {
+        message: 'Resource not found',
+        suggestion: 'Verify the project ID and collection path are correct',
+    },
+
+    // Quota errors
+    'resource-exhausted': {
+        message: 'Quota exceeded',
+        suggestion: 'Try reducing --batch-size or --parallel, or wait and retry later',
+    },
+    RESOURCE_EXHAUSTED: {
+        message: 'Quota exceeded',
+        suggestion: 'Try reducing --batch-size or --parallel, or wait and retry later',
+    },
+
+    // Invalid argument errors
+    'invalid-argument': {
+        message: 'Invalid argument',
+        suggestion: 'Check your query filters and document data',
+    },
+    INVALID_ARGUMENT: {
+        message: 'Invalid argument',
+        suggestion: 'Check your query filters and document data',
+    },
+
+    // Deadline exceeded
+    'deadline-exceeded': {
+        message: 'Request timeout',
+        suggestion: 'Try reducing --batch-size or check your network connection',
+    },
+    DEADLINE_EXCEEDED: {
+        message: 'Request timeout',
+        suggestion: 'Try reducing --batch-size or check your network connection',
+    },
+
+    // Already exists
+    'already-exists': {
+        message: 'Document already exists',
+        suggestion: 'Use --merge option to update existing documents',
+    },
+    ALREADY_EXISTS: {
+        message: 'Document already exists',
+        suggestion: 'Use --merge option to update existing documents',
+    },
+
+    // Aborted
+    aborted: {
+        message: 'Operation aborted',
+        suggestion: 'A concurrent operation conflicted. Retry the transfer',
+    },
+    ABORTED: {
+        message: 'Operation aborted',
+        suggestion: 'A concurrent operation conflicted. Retry the transfer',
+    },
+};
+
+export function formatFirebaseError(error: Error & { code?: string }): FirebaseErrorInfo {
+    // Check by error code first
+    if (error.code) {
+        const mapped = errorMap[error.code];
+        if (mapped) {
+            return mapped;
+        }
+    }
+
+    // Check by error message keywords
+    const message = error.message.toLowerCase();
+
+    if (message.includes('credential') || message.includes('authentication')) {
+        return errorMap['app/invalid-credential'];
+    }
+    if (message.includes('permission') || message.includes('denied')) {
+        return errorMap['permission-denied'];
+    }
+    if (message.includes('unavailable') || message.includes('network')) {
+        return errorMap['unavailable'];
+    }
+    if (message.includes('not found') || message.includes('not_found')) {
+        return errorMap['not-found'];
+    }
+    if (message.includes('quota') || message.includes('exhausted') || message.includes('rate')) {
+        return errorMap['resource-exhausted'];
+    }
+    if (message.includes('timeout') || message.includes('deadline')) {
+        return errorMap['deadline-exceeded'];
+    }
+
+    // Default: return original message
+    return {
+        message: error.message,
+    };
+}
+
+export function logFirebaseError(
+    error: Error & { code?: string },
+    context: string,
+    logger?: { error: (msg: string, data?: Record<string, unknown>) => void }
+): void {
+    const info = formatFirebaseError(error);
+
+    console.error(`\n❌ ${context}: ${info.message}`);
+    if (info.suggestion) {
+        console.error(`   Hint: ${info.suggestion}`);
+    }
+    if (error.code) {
+        console.error(`   Code: ${error.code}`);
+    }
+
+    if (logger) {
+        logger.error(`${context}: ${info.message}`, {
+            code: error.code,
+            originalMessage: error.message,
+            suggestion: info.suggestion,
+        });
+    }
+}

package/src/utils/index.ts

@@ -0,0 +1,7 @@
+export { withRetry, type RetryOptions } from './retry.js';
+export { Logger } from './logger.js';
+export { checkCredentialsExist, ensureCredentials } from './credentials.js';
+export { matchesExcludePattern } from './patterns.js';
+export { formatFirebaseError, logFirebaseError, type FirebaseErrorInfo } from './errors.js';
+export { RateLimiter } from './rate-limiter.js';
+export { estimateDocumentSize, formatBytes, FIRESTORE_MAX_DOC_SIZE } from './doc-size.js';

package/src/utils/logger.ts

@@ -0,0 +1,61 @@
+import fs from 'node:fs';
+import type { Stats, LogEntry } from '../types.js';
+
+export class Logger {
+    private readonly logPath: string | undefined;
+    private readonly entries: LogEntry[] = [];
+    private readonly startTime: Date;
+
+    constructor(logPath?: string) {
+        this.logPath = logPath;
+        this.startTime = new Date();
+    }
+
+    log(level: string, message: string, data: Record<string, unknown> = {}): void {
+        const entry: LogEntry = {
+            timestamp: new Date().toISOString(),
+            level,
+            message,
+            ...data,
+        };
+        this.entries.push(entry);
+
+        if (this.logPath) {
+            const line =
+                `[${entry.timestamp}] [${level}] ${message}` +
+                (Object.keys(data).length > 0 ? ` ${JSON.stringify(data)}` : '') +
+                '\n';
+            fs.appendFileSync(this.logPath, line);
+        }
+    }
+
+    info(message: string, data?: Record<string, unknown>): void {
+        this.log('INFO', message, data);
+    }
+
+    error(message: string, data?: Record<string, unknown>): void {
+        this.log('ERROR', message, data);
+    }
+
+    success(message: string, data?: Record<string, unknown>): void {
+        this.log('SUCCESS', message, data);
+    }
+
+    init(): void {
+        if (this.logPath) {
+            const header = `# fscopy transfer log\n# Started: ${this.startTime.toISOString()}\n\n`;
+            fs.writeFileSync(this.logPath, header);
+        }
+    }
+
+    summary(stats: Stats, duration: string): void {
+        if (this.logPath) {
+            let summary = `\n# Summary\n# Collections: ${stats.collectionsProcessed}\n`;
+            if (stats.documentsDeleted > 0) {
+                summary += `# Deleted: ${stats.documentsDeleted}\n`;
+            }
+            summary += `# Transferred: ${stats.documentsTransferred}\n# Errors: ${stats.errors}\n# Duration: ${duration}s\n`;
+            fs.appendFileSync(this.logPath, summary);
+        }
+    }
+}

package/src/utils/patterns.ts

@@ -0,0 +1,13 @@
+export function matchesExcludePattern(path: string, patterns: string[]): boolean {
+    for (const pattern of patterns) {
+        if (pattern.includes('*')) {
+            // Convert glob pattern to regex
+            const regex = new RegExp('^' + pattern.replaceAll('*', '.*') + '$');
+            if (regex.test(path)) return true;
+        } else if (path === pattern || path.endsWith('/' + pattern)) {
+            // Exact match or ends with pattern
+            return true;
+        }
+    }
+    return false;
+}

package/src/utils/rate-limiter.ts

@@ -0,0 +1,62 @@
+/**
+ * Simple rate limiter using token bucket algorithm
+ * Limits the rate of operations per second
+ */
+export class RateLimiter {
+    private tokens: number;
+    private lastRefill: number;
+    private readonly maxTokens: number;
+    private readonly refillRate: number; // tokens per ms
+
+    /**
+     * Create a rate limiter
+     * @param docsPerSecond Maximum documents per second (0 = unlimited)
+     */
+    constructor(docsPerSecond: number) {
+        this.maxTokens = docsPerSecond;
+        this.tokens = docsPerSecond;
+        this.lastRefill = Date.now();
+        this.refillRate = docsPerSecond / 1000; // tokens per ms
+    }
+
+    /**
+     * Check if rate limiting is enabled
+     */
+    isEnabled(): boolean {
+        return this.maxTokens > 0;
+    }
+
+    /**
+     * Wait until we can proceed with the given number of operations
+     * @param count Number of operations to perform
+     */
+    async acquire(count: number = 1): Promise<void> {
+        if (!this.isEnabled()) return;
+
+        // Refill tokens based on time elapsed
+        const now = Date.now();
+        const elapsed = now - this.lastRefill;
+        this.tokens = Math.min(this.maxTokens, this.tokens + elapsed * this.refillRate);
+        this.lastRefill = now;
+
+        // If we have enough tokens, consume and proceed
+        if (this.tokens >= count) {
+            this.tokens -= count;
+            return;
+        }
+
+        // Wait for tokens to be available
+        const tokensNeeded = count - this.tokens;
+        const waitTime = Math.ceil(tokensNeeded / this.refillRate);
+
+        await this.sleep(waitTime);
+
+        // After waiting, we should have enough tokens
+        this.tokens = 0; // We consumed them all
+        this.lastRefill = Date.now();
+    }
+
+    private sleep(ms: number): Promise<void> {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}

package/src/utils/retry.ts

@@ -0,0 +1,29 @@
+export interface RetryOptions {
+    retries?: number;
+    baseDelay?: number;
+    maxDelay?: number;
+    onRetry?: (attempt: number, max: number, error: Error, delay: number) => void;
+}
+
+export async function withRetry<T>(fn: () => Promise<T>, options: RetryOptions = {}): Promise<T> {
+    const { retries = 3, baseDelay = 1000, maxDelay = 30000, onRetry } = options;
+
+    let lastError: Error | undefined;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        } catch (error) {
+            lastError = error as Error;
+
+            if (attempt < retries) {
+                const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
+                if (onRetry) {
+                    onRetry(attempt + 1, retries, lastError, delay);
+                }
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+    }
+
+    throw lastError;
+}