@fazetitans/fscopy 1.3.1 → 1.5.0

package/src/types.ts

@@ -5,7 +5,7 @@
 export interface WhereFilter {
     field: string;
     operator: FirebaseFirestore.WhereFilterOp;
-    value: string | number | boolean;
+    value: string | number | boolean | null;
 }
 
 export interface Config {
@@ -38,6 +38,7 @@ export interface Config {
     detectConflicts: boolean;
     maxDepth: number;
     verifyIntegrity: boolean;
+    allowHttpWebhook: boolean;
 }
 
 // Config after validation - required fields are guaranteed non-null
@@ -97,7 +98,7 @@ export interface CliArgs {
     yes: boolean;
     log?: string;
     maxLogSize?: string;
-    retries: number;
+    retries?: number;
     quiet: boolean;
     where?: string[];
     exclude?: string[];
@@ -121,5 +122,6 @@ export interface CliArgs {
     detectConflicts?: boolean;
     maxDepth?: number;
     verifyIntegrity?: boolean;
+    allowHttpWebhook?: boolean;
     validateOnly?: boolean;
 }

package/src/utils/credentials.ts

@@ -28,10 +28,13 @@ export function ensureCredentials(): void {
     const { exists, path: credPath } = checkCredentialsExist();
 
     if (!exists) {
-        console.error('\n❌ Google Cloud credentials not found.');
-        console.error(`   Expected at: ${credPath}\n`);
-        console.error('   Run this command to authenticate:');
-        console.error('   gcloud auth application-default login\n');
+        const msg = [
+            '\n❌ Google Cloud credentials not found.',
+            `   Expected at: ${credPath}\n`,
+            '   Run this command to authenticate:',
+            '   gcloud auth application-default login\n',
+        ].join('\n');
+        process.stderr.write(msg);
         process.exit(1);
     }
 }

package/src/utils/errors.ts

@@ -1,3 +1,8 @@
+/** Error with optional Firebase/gRPC error code */
+export interface FirebaseError extends Error {
+    code?: string;
+}
+
 export interface FirebaseErrorInfo {
     message: string;
     suggestion?: string;
@@ -95,7 +100,7 @@ const errorMap: Record<string, FirebaseErrorInfo> = {
     },
 };
 
-export function formatFirebaseError(error: Error & { code?: string }): FirebaseErrorInfo {
+export function formatFirebaseError(error: FirebaseError): FirebaseErrorInfo {
     // Check by error code first
     if (error.code) {
         const mapped = errorMap[error.code];
@@ -133,7 +138,7 @@ export function formatFirebaseError(error: Error & { code?: string }): FirebaseE
 }
 
 export function logFirebaseError(
-    error: Error & { code?: string },
+    error: FirebaseError,
     context: string,
     logger?: { error: (msg: string, data?: Record<string, unknown>) => void }
 ): void {

package/src/utils/index.ts

@@ -3,6 +3,11 @@ export { Output, type OutputOptions } from './output.js';
 export { ProgressBarWrapper, type ProgressBarOptions } from './progress.js';
 export { checkCredentialsExist, ensureCredentials } from './credentials.js';
 export { matchesExcludePattern } from './patterns.js';
-export { formatFirebaseError, logFirebaseError, type FirebaseErrorInfo } from './errors.js';
+export {
+    formatFirebaseError,
+    logFirebaseError,
+    type FirebaseError,
+    type FirebaseErrorInfo,
+} from './errors.js';
 export { RateLimiter } from './rate-limiter.js';
 export { estimateDocumentSize, formatBytes, FIRESTORE_MAX_DOC_SIZE } from './doc-size.js';

package/src/utils/output.ts

@@ -1,4 +1,5 @@
 import fs from 'node:fs';
+import path from 'node:path';
 import { SEPARATOR_LENGTH } from '../constants.js';
 import type { Stats, LogEntry } from '../types.js';
 import { rotateFileIfNeeded } from './file-rotation.js';
@@ -62,6 +63,11 @@ export class Output {
 
     init(): void {
         if (this.options.logFile) {
+            // Ensure parent directory exists
+            const dir = path.dirname(this.options.logFile);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
             this.rotateLogIfNeeded();
             const header = `# fscopy transfer log\n# Started: ${this.startTime.toISOString()}\n\n`;
             fs.writeFileSync(this.options.logFile, header);

package/src/utils/progress.ts

@@ -135,6 +135,16 @@ export class ProgressBarWrapper {
         }
     }
 
+    /**
+     * Increase the progress bar total by the given amount.
+     * Used for dynamically discovered work (e.g., subcollections).
+     */
+    addToTotal(count: number): void {
+        if (this.bar && count > 0) {
+            this.bar.setTotal(this.bar.getTotal() + count);
+        }
+    }
+
     /**
      * Check if the progress bar is active.
      */

package/src/webhook/index.ts

@@ -1,5 +1,11 @@
 import type { Stats } from '../types.js';
 import type { Output } from '../utils/output.js';
+import {
+    WEBHOOK_TIMEOUT_MS,
+    WEBHOOK_MAX_PAYLOAD_BYTES,
+    WEBHOOK_MAX_RETRIES,
+    WEBHOOK_RETRY_DELAY_MS,
+} from '../constants.js';
 
 export interface WebhookPayload {
     source: string;
@@ -22,12 +28,21 @@ export function detectWebhookType(url: string): 'slack' | 'discord' | 'custom' {
     return 'custom';
 }
 
-export function validateWebhookUrl(url: string): { valid: boolean; warning?: string } {
+export function validateWebhookUrl(
+    url: string,
+    allowHttp: boolean = false
+): { valid: boolean; warning?: string } {
     try {
         const parsed = new URL(url);
         const isLocalhost = parsed.hostname === 'localhost' || parsed.hostname === '127.0.0.1';
 
         if (parsed.protocol !== 'https:' && !isLocalhost) {
+            if (!allowHttp) {
+                return {
+                    valid: false,
+                    warning: `Webhook URL uses HTTP instead of HTTPS. Use --allow-http-webhook to allow unencrypted webhooks.`,
+                };
+            }
             return {
                 valid: true,
                 warning: `Webhook URL uses HTTP instead of HTTPS. Data will be sent unencrypted.`,
@@ -105,33 +120,19 @@ export function formatDiscordPayload(payload: WebhookPayload): Record<string, un
     };
 }
 
-export async function sendWebhook(
+async function attemptWebhookSend(
     webhookUrl: string,
-    payload: WebhookPayload,
+    bodyJson: string,
     output: Output
-): Promise<void> {
-    const webhookType = detectWebhookType(webhookUrl);
-
-    let body: Record<string, unknown>;
-    switch (webhookType) {
-        case 'slack':
-            body = formatSlackPayload(payload);
-            break;
-        case 'discord':
-            body = formatDiscordPayload(payload);
-            break;
-        default:
-            body = payload as unknown as Record<string, unknown>;
-    }
+): Promise<boolean> {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), WEBHOOK_TIMEOUT_MS);
 
     try {
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), 30000);
-
         const response = await fetch(webhookUrl, {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify(body),
+            body: bodyJson,
             signal: controller.signal,
         });
 
@@ -142,7 +143,6 @@ export async function sendWebhook(
             const statusCode = response.status;
 
             if (statusCode >= 400 && statusCode < 500) {
-                // Client error - likely bad URL or payload format
                 output.logError(`Webhook client error (${statusCode})`, {
                     url: webhookUrl,
                     status: statusCode,
@@ -151,36 +151,119 @@ export async function sendWebhook(
                 output.warn(
                     `⚠️  Webhook failed (HTTP ${statusCode}): Check webhook URL or payload format`
                 );
-            } else if (statusCode >= 500) {
-                // Server error - retry might help
+                // Client errors are not retryable
+                return false;
+            }
+
+            if (statusCode >= 500) {
                 output.logError(`Webhook server error (${statusCode})`, {
                     url: webhookUrl,
                     status: statusCode,
                     error: errorText,
                 });
-                output.warn(
-                    `⚠️  Webhook server error (HTTP ${statusCode}): The webhook service may be temporarily unavailable`
-                );
+                throw new Error(`Server error (HTTP ${statusCode})`);
             }
-            return;
+
+            return false;
         }
 
-        output.logInfo(`Webhook sent successfully (${webhookType})`, { url: webhookUrl });
-        output.info(`📤 Webhook notification sent (${webhookType})`);
-    } catch (error) {
-        const err = error as Error;
-
-        if (err.name === 'AbortError') {
-            output.logError('Webhook timeout after 30s', { url: webhookUrl });
-            output.warn('⚠️  Webhook request timed out after 30 seconds');
-        } else if (err.message.includes('ECONNREFUSED') || err.message.includes('ENOTFOUND')) {
-            output.logError(`Webhook connection failed: ${err.message}`, { url: webhookUrl });
-            output.warn(
-                `⚠️  Webhook connection failed: Unable to reach ${new URL(webhookUrl).hostname}`
-            );
-        } else {
-            output.logError(`Failed to send webhook: ${err.message}`, { url: webhookUrl });
-            output.warn(`⚠️  Failed to send webhook: ${err.message}`);
+        return true;
+    } finally {
+        clearTimeout(timeout);
+    }
+}
+
+export async function sendWebhook(
+    webhookUrl: string,
+    payload: WebhookPayload,
+    output: Output
+): Promise<boolean> {
+    const webhookType = detectWebhookType(webhookUrl);
+
+    let body: Record<string, unknown>;
+    switch (webhookType) {
+        case 'slack':
+            body = formatSlackPayload(payload);
+            break;
+        case 'discord':
+            body = formatDiscordPayload(payload);
+            break;
+        default:
+            body = { ...payload };
+    }
+
+    const bodyJson = JSON.stringify(body);
+
+    // Validate payload size
+    const payloadSize = new TextEncoder().encode(bodyJson).length;
+    if (payloadSize > WEBHOOK_MAX_PAYLOAD_BYTES) {
+        output.logError('Webhook payload too large', {
+            url: webhookUrl,
+            size: payloadSize,
+            limit: WEBHOOK_MAX_PAYLOAD_BYTES,
+        });
+        output.warn(
+            `⚠️  Webhook payload too large (${Math.round(payloadSize / 1024)}KB > ${Math.round(WEBHOOK_MAX_PAYLOAD_BYTES / 1024)}KB limit)`
+        );
+        return false;
+    }
+
+    // Retry loop for server errors and network failures
+    for (let attempt = 0; attempt <= WEBHOOK_MAX_RETRIES; attempt++) {
+        try {
+            const success = await attemptWebhookSend(webhookUrl, bodyJson, output);
+            if (success) {
+                output.logInfo(`Webhook sent successfully (${webhookType})`, { url: webhookUrl });
+                output.info(`📤 Webhook notification sent (${webhookType})`);
+                return true;
+            }
+            // Client error (4xx) - don't retry
+            return false;
+        } catch (error) {
+            const err = error as Error;
+            const isLastAttempt = attempt === WEBHOOK_MAX_RETRIES;
+
+            if (err.name === 'AbortError') {
+                output.logError(`Webhook timeout after ${WEBHOOK_TIMEOUT_MS / 1000}s`, {
+                    url: webhookUrl,
+                    attempt: attempt + 1,
+                });
+                if (isLastAttempt) {
+                    output.warn(
+                        `⚠️  Webhook request timed out after ${WEBHOOK_TIMEOUT_MS / 1000} seconds`
+                    );
+                    return false;
+                }
+            } else if (err.message.includes('ECONNREFUSED') || err.message.includes('ENOTFOUND')) {
+                output.logError(`Webhook connection failed: ${err.message}`, {
+                    url: webhookUrl,
+                    attempt: attempt + 1,
+                });
+                if (isLastAttempt) {
+                    output.warn(
+                        `⚠️  Webhook connection failed: Unable to reach ${new URL(webhookUrl).hostname}`
+                    );
+                    return false;
+                }
+            } else {
+                output.logError(`Failed to send webhook: ${err.message}`, {
+                    url: webhookUrl,
+                    attempt: attempt + 1,
+                });
+                if (isLastAttempt) {
+                    output.warn(`⚠️  Failed to send webhook: ${err.message}`);
+                    return false;
+                }
+            }
+
+            // Wait before retrying
+            const delay = WEBHOOK_RETRY_DELAY_MS * Math.pow(2, attempt);
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            output.logInfo(`Retrying webhook (attempt ${attempt + 2}/${WEBHOOK_MAX_RETRIES + 1})`, {
+                url: webhookUrl,
+            });
         }
     }
+
+    return false;
 }