flow-frame-core 0.1.0

package/dist/capture_training_data_endpoints.js

@@ -0,0 +1,1812 @@
+import fs from 'fs';
+import path from 'path';
+import { console_log_it, downloadFile } from './utils.js';
+import ffmpeg from 'fluent-ffmpeg';
+import sharp from 'sharp';
+import admin from 'firebase-admin';
+import serviceAccount from './flowframe-auto-firebase-adminsdk.json' with { type: "json" };
+admin.initializeApp({
+    credential: admin.credential.cert(serviceAccount),
+    databaseURL: "https://flowframe-auto-default-rtdb.firebaseio.com",
+    storageBucket: "flowframe-auto.firebasestorage.app"
+});
+/**
+ * Recursively list all files under a root directory.
+ * @param {string} rootDir - The starting directory.
+ * @param {{ followSymlinks?: boolean }} [opts]
+ * @returns {Promise<string[]>} Absolute file paths.
+ */
+export async function listFiles(rootDir, opts = {}) {
+    const { followSymlinks = false } = opts;
+    const start = path.resolve(rootDir);
+    const out = [];
+    const { readdir, stat } = fs.promises;
+    async function walk(dir) {
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return; // unreadable dir — skip
+        }
+        await Promise.all(entries.map(async (ent) => {
+            const full = path.join(dir, ent.name);
+            if (ent.isDirectory()) {
+                await walk(full);
+            }
+            else if (ent.isFile()) {
+                out.push(full);
+            }
+            else if (followSymlinks && ent.isSymbolicLink()) {
+                try {
+                    const s = await stat(full);
+                    if (s.isDirectory())
+                        await walk(full);
+                    else if (s.isFile())
+                        out.push(full);
+                }
+                catch {
+                    // broken/unreadable symlink — skip
+                }
+            }
+        }));
+    }
+    await walk(start);
+    return out;
+}
+/**
+ * Cleans up training directories by keeping only level_#/runs/exp folders and deleting other paths
+ * @param {string} rootPath - Root directory to scan for level folders
+ * @param {Object} options - Cleanup options
+ * @param {boolean} options.dryRun - If true, only logs what would be deleted without actually deleting (default: false)
+ * @param {Function} options.onProgress - Progress callback function
+ * @param {Array<string>} options.preservePaths - Additional paths to preserve (relative to level folder)
+ * @returns {Promise<Object>} Cleanup result with statistics
+ */
+async function cleanupTrainingDirectories(rootPath, options = {}) {
+    const { dryRun = false, onProgress = null, preservePaths = ['runs/exp'] // Always preserve runs/exp
+     } = options;
+    console_log_it(`Starting cleanup of training directories in: ${rootPath}`, 'cleanup_training_directories');
+    if (!fs.existsSync(rootPath)) {
+        throw new Error(`Root path does not exist: ${rootPath}`);
+    }
+    const stats = {
+        levelFoldersFound: 0,
+        pathsPreserved: 0,
+        pathsDeleted: 0,
+        spaceFreed: 0,
+        errors: []
+    };
+    try {
+        // Find all level_# folders
+        const files = await listFiles(rootPath);
+        for (let i = 0; i < files.length; i++) {
+            const file = files[i];
+            if (file.includes('runs/exp') || file.includes('annotations.json'))
+                continue;
+            // Process each file as needed
+            //delete the files.
+            if (!dryRun) {
+                fs.unlinkSync(file);
+            }
+            stats.pathsDeleted++;
+            if (onProgress) {
+                onProgress({
+                    type: 'start',
+                    levelFoldersFound: stats.levelFoldersFound,
+                    dryRun
+                });
+            }
+        }
+        if (onProgress) {
+            onProgress({
+                type: 'complete',
+                result: {
+                    success: true,
+                    dryRun,
+                    rootPath,
+                    stats: {
+                        ...stats,
+                        spaceFreedMB: (stats.spaceFreed / 1024 / 1024).toFixed(2),
+                        spaceFreedGB: (stats.spaceFreed / 1024 / 1024 / 1024).toFixed(2)
+                    },
+                    message: `Cleanup ${dryRun ? 'simulation' : 'completed'}. Preserved: ${stats.pathsPreserved}, Deleted: ${stats.pathsDeleted}, Freed: ${(stats.spaceFreed / 1024 / 1024).toFixed(2)} MB`
+                }
+            });
+        }
+        return true;
+    }
+    catch (error) {
+        console_log_it(`Cleanup failed: ${error.message}`, 'cleanup_training_directories');
+        throw error;
+    }
+}
+/**
+ * Recursively processes subdirectories to preserve only specified paths
+ */
+async function cleanupDirectoryRecursive(dirPath, preservePaths, currentRelativePath, stats, dryRun) {
+    try {
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const entryPath = path.join(dirPath, entry.name);
+            const newRelativePath = path.join(currentRelativePath, entry.name).replace(/\\/g, '/');
+            // Check if this path should be preserved
+            const shouldPreserve = preservePaths.some(preservePath => {
+                return preservePath.startsWith(newRelativePath) || newRelativePath === preservePath;
+            });
+            if (shouldPreserve) {
+                stats.pathsPreserved++;
+                // If this is a directory that's part of a preserve path, continue recursively
+                if (entry.isDirectory()) {
+                    const matchingPreservePaths = preservePaths.filter(p => p.startsWith(newRelativePath + '/'));
+                    if (matchingPreservePaths.length > 0) {
+                        await cleanupDirectoryRecursive(entryPath, matchingPreservePaths, newRelativePath, stats, dryRun);
+                    }
+                }
+            }
+            else {
+                // Calculate size and delete
+                let size = 0;
+                try {
+                    if (entry.isDirectory()) {
+                        size = await getDirectorySize(entryPath);
+                    }
+                    else {
+                        const stat = fs.statSync(entryPath);
+                        size = stat.size;
+                    }
+                }
+                catch (sizeError) {
+                    console_log_it(`Warning: Could not calculate size of ${entryPath}`, 'cleanup_training_directories');
+                }
+                if (!dryRun) {
+                    try {
+                        if (entry.isDirectory()) {
+                            fs.rmSync(entryPath, { recursive: true, force: true });
+                        }
+                        else {
+                            fs.unlinkSync(entryPath);
+                        }
+                    }
+                    catch (deleteError) {
+                        stats.errors.push({
+                            path: entryPath,
+                            error: deleteError.message
+                        });
+                        continue;
+                    }
+                }
+                stats.pathsDeleted++;
+                stats.spaceFreed += size;
+            }
+        }
+    }
+    catch (error) {
+        stats.errors.push({
+            path: dirPath,
+            error: error.message
+        });
+    }
+}
+/**
+ * Calculates the total size of a directory recursively
+ */
+async function getDirectorySize(dirPath) {
+    let totalSize = 0;
+    try {
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const entryPath = path.join(dirPath, entry.name);
+            try {
+                if (entry.isDirectory()) {
+                    totalSize += await getDirectorySize(entryPath);
+                }
+                else {
+                    const stat = fs.statSync(entryPath);
+                    totalSize += stat.size;
+                }
+            }
+            catch (error) {
+                // Skip files that can't be accessed
+                continue;
+            }
+        }
+    }
+    catch (error) {
+        // Return 0 if directory can't be read
+        return 0;
+    }
+    return totalSize;
+}
+// Database connection management
+let databaseConnectionActive = true;
+let reconnectPromise = null;
+/**
+ * Disconnects from Firebase Realtime Database to save resources during long operations
+ * @returns {Promise<void>}
+ */
+async function disconnectFromDatabase() {
+    if (!databaseConnectionActive) {
+        console.log('Database already disconnected', 'database_management');
+        return;
+    }
+    try {
+        console.log('Disconnecting from Firebase Realtime Database...', 'database_management');
+        await admin.database().goOffline();
+        databaseConnectionActive = false;
+        console.log('Successfully disconnected from Firebase Realtime Database', 'database_management');
+    }
+    catch (error) {
+        console.error('Error disconnecting from database:', error.message, 'database_management');
+        throw error;
+    }
+}
+/**
+ * Reconnects to Firebase Realtime Database after long operations
+ * @returns {Promise<void>}
+ */
+async function reconnectToDatabase() {
+    if (databaseConnectionActive) {
+        console.log('Database already connected', 'database_management');
+        return;
+    }
+    // Prevent multiple reconnection attempts
+    if (reconnectPromise) {
+        console.log('Reconnection already in progress, waiting...', 'database_management');
+        return await reconnectPromise;
+    }
+    reconnectPromise = (async () => {
+        try {
+            console.log('Reconnecting to Firebase Realtime Database...', 'database_management');
+            await admin.database().goOnline();
+            databaseConnectionActive = true;
+            console.log('Successfully reconnected to Firebase Realtime Database', 'database_management');
+        }
+        catch (error) {
+            console.error('Error reconnecting to database:', error.message, 'database_management');
+            throw error;
+        }
+        finally {
+            reconnectPromise = null;
+        }
+    })();
+    return await reconnectPromise;
+}
+/**
+ * Executes a function with database disconnected, then reconnects
+ * @param {Function} asyncOperation - The async operation to execute while disconnected
+ * @param {Object} options - Options for the operation
+ * @param {boolean} options.autoReconnect - Whether to automatically reconnect after operation (default: true)
+ * @returns {Promise<any>} Result of the operation
+ */
+async function executeWithDatabaseDisconnected(asyncOperation, options = {}) {
+    const { autoReconnect = true } = options;
+    const wasConnected = databaseConnectionActive;
+    try {
+        if (wasConnected) {
+            await disconnectFromDatabase();
+        }
+        console.log('Executing operation with database disconnected...', 'database_management');
+        const result = await asyncOperation();
+        console.log('Operation completed successfully', 'database_management');
+        return result;
+    }
+    catch (error) {
+        console.error('Operation failed while database disconnected:', error.message, 'database_management');
+        throw error;
+    }
+    finally {
+        if (wasConnected && autoReconnect) {
+            try {
+                await reconnectToDatabase();
+            }
+            catch (reconnectError) {
+                console.error('Failed to reconnect to database after operation:', reconnectError.message, 'database_management');
+                // Don't throw here as the main operation might have succeeded
+            }
+        }
+    }
+}
+/**
+ * Safe database operation that handles connection state
+ * @param {Function} dbOperation - Database operation function
+ * @param {Object} options - Options for the operation
+ * @param {number} options.maxRetries - Maximum number of retry attempts (default: 3)
+ * @param {number} options.retryDelay - Delay between retries in ms (default: 1000)
+ * @returns {Promise<any>} Result of the database operation
+ */
+async function safeDatabaseOperation(dbOperation, options = {}) {
+    const { maxRetries = 3, retryDelay = 1000 } = options;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            // Ensure we're connected before attempting the operation
+            if (!databaseConnectionActive) {
+                await reconnectToDatabase();
+            }
+            return await dbOperation();
+        }
+        catch (error) {
+            console.error(`Database operation attempt ${attempt} failed:`, error.message, 'database_management');
+            if (attempt === maxRetries) {
+                throw error;
+            }
+            // If it's a connection error, try to reconnect
+            if (error.message.includes('offline') || error.message.includes('connection')) {
+                console.log('Connection error detected, attempting to reconnect...', 'database_management');
+                databaseConnectionActive = false;
+                try {
+                    await reconnectToDatabase();
+                }
+                catch (reconnectError) {
+                    console.error('Reconnection failed:', reconnectError.message, 'database_management');
+                }
+            }
+            // Wait before retrying
+            await new Promise(resolve => setTimeout(resolve, retryDelay * attempt));
+        }
+    }
+}
+// Progress throttling system to prevent Firestore contention
+const progressThrottlers = new Map();
+/**
+ * Creates a throttled progress updater to prevent database contention
+ * @param {string} subJobId - The subjob ID
+ * @param {Function} updateSubJobFn - The x function to use
+ * @param {number} throttleMs - Minimum milliseconds between updates (default: 2000)
+ * @returns {Function} Throttled update function
+ */
+function createThrottledProgressUpdater(subJobId, updateSubJobFn, throttleMs = 2000) {
+    if (progressThrottlers.has(subJobId)) {
+        return progressThrottlers.get(subJobId);
+    }
+    let lastUpdate = 0;
+    let lastPercent = -1;
+    let pendingUpdate = null;
+    let isUpdating = false;
+    const throttledUpdater = async (updateData) => {
+        const now = Date.now();
+        const currentPercent = Math.floor(updateData.percent || 0);
+        // Always update if it's a significant change (5% or more) or enough time has passed
+        const significantChange = Math.abs(currentPercent - lastPercent) >= 5;
+        const enoughTimePassed = (now - lastUpdate) >= throttleMs;
+        if (significantChange || enoughTimePassed || updateData.status || updateData.message) {
+            // Cancel any pending update since we're doing one now
+            if (pendingUpdate) {
+                clearTimeout(pendingUpdate);
+                pendingUpdate = null;
+            }
+            if (!isUpdating) {
+                isUpdating = true;
+                try {
+                    await safeDatabaseOperation(async () => {
+                        await updateSubJobFn(subJobId, updateData);
+                    });
+                    lastUpdate = now;
+                    lastPercent = currentPercent;
+                }
+                catch (error) {
+                    console.error(`Failed to update progress for subjob ${subJobId}:`, error.message);
+                }
+                finally {
+                    isUpdating = false;
+                }
+            }
+        }
+        else {
+            // Schedule a delayed update for the latest data
+            if (pendingUpdate) {
+                clearTimeout(pendingUpdate);
+            }
+            pendingUpdate = setTimeout(async () => {
+                if (!isUpdating) {
+                    isUpdating = true;
+                    try {
+                        await safeDatabaseOperation(async () => {
+                            await updateSubJobFn(subJobId, updateData);
+                        });
+                        lastUpdate = Date.now();
+                        lastPercent = Math.floor(updateData.percent || 0);
+                    }
+                    catch (error) {
+                        console.error(`Failed to update delayed progress for subjob ${subJobId}:`, error.message);
+                    }
+                    finally {
+                        isUpdating = false;
+                    }
+                }
+                pendingUpdate = null;
+            }, throttleMs);
+        }
+    };
+    // Cleanup function
+    throttledUpdater.cleanup = () => {
+        if (pendingUpdate) {
+            clearTimeout(pendingUpdate);
+            pendingUpdate = null;
+        }
+        progressThrottlers.delete(subJobId);
+    };
+    progressThrottlers.set(subJobId, throttledUpdater);
+    return throttledUpdater;
+}
+/**
+ * Cleans up throttled progress updater for a subjob
+ * @param {string} subJobId - The subjob ID
+ */
+function cleanupProgressThrottler(subJobId) {
+    const throttler = progressThrottlers.get(subJobId);
+    if (throttler && throttler.cleanup) {
+        throttler.cleanup();
+    }
+}
+// Get Firebase Storage bucket
+const bucketContext = {};
+function getBucket() {
+    if (!bucketContext.bucket) {
+        bucketContext.bucket = admin.storage().bucket();
+    }
+    return bucketContext.bucket;
+}
+/**
+ * Tests if the Firebase Storage bucket exists and is accessible
+ * @param {string} bucketName - Optional bucket name to test (uses default if not provided)
+ * @returns {Promise<Object>} Test result with bucket information
+ */
+async function testStorageBucketExists(bucketName = null) {
+    try {
+        console.log('Testing storage bucket existence and accessibility', 'test_storage_bucket');
+        let bucket;
+        let testBucketName;
+        if (bucketName) {
+            // Test specific bucket
+            bucket = admin.storage().bucket(bucketName);
+            testBucketName = bucketName;
+        }
+        else {
+            // Test default bucket
+            bucket = getBucket();
+            testBucketName = bucket.name;
+        }
+        console.log(`Testing bucket: ${testBucketName}`, 'test_storage_bucket');
+        // Test 1: Check if bucket exists
+        const [exists] = await bucket.exists();
+        if (!exists) {
+            console.log(`Bucket does not exist: ${testBucketName}`, 'test_storage_bucket');
+            return {
+                exists: false,
+                accessible: false,
+                bucketName: testBucketName,
+                error: 'Bucket does not exist',
+                tests: {
+                    existence: false,
+                    metadata: false,
+                    listFiles: false,
+                    writeAccess: false
+                }
+            };
+        }
+        console.log(`Bucket exists: ${testBucketName}`, 'test_storage_bucket');
+        // Test 2: Get bucket metadata
+        let metadata = null;
+        let metadataAccessible = false;
+        try {
+            [metadata] = await bucket.getMetadata();
+            metadataAccessible = true;
+            console.log(`Bucket metadata accessible. Location: ${metadata.location}, Storage class: ${metadata.storageClass}`, 'test_storage_bucket');
+        }
+        catch (metadataError) {
+            console.log(`Cannot access bucket metadata: ${metadataError.message}`, 'test_storage_bucket');
+        }
+        // Test 3: Test list files permission (with minimal query)
+        let canListFiles = false;
+        let fileCount = 0;
+        try {
+            const [files] = await bucket.getFiles({ maxResults: 1 });
+            canListFiles = true;
+            // Get actual count with a separate query
+            try {
+                const [allFiles] = await bucket.getFiles({ maxResults: 100 });
+                fileCount = allFiles.length;
+            }
+            catch (countError) {
+                fileCount = 'unknown';
+            }
+            console.log(`Can list files in bucket. Sample files found: ${fileCount}`, 'test_storage_bucket');
+        }
+        catch (listError) {
+            console.log(`Cannot list files in bucket: ${listError.message}`, 'test_storage_bucket');
+        }
+        // Test 4: Test write access by creating a test file
+        let canWrite = false;
+        let writeError = null;
+        const testFileName = `test_access_${Date.now()}.txt`;
+        const testFilePath = `test/${testFileName}`;
+        try {
+            const testFile = bucket.file(testFilePath);
+            await testFile.save('test content for bucket access verification', {
+                metadata: {
+                    contentType: 'text/plain',
+                    metadata: {
+                        purpose: 'bucket_access_test',
+                        createdAt: new Date().toISOString()
+                    }
+                }
+            });
+            // Verify the file was created
+            const [createdExists] = await testFile.exists();
+            if (createdExists) {
+                canWrite = true;
+                console.log(`Write access confirmed. Test file created: ${testFilePath}`, 'test_storage_bucket');
+                // Clean up test file
+                try {
+                    await testFile.delete();
+                    console.log(`Test file cleaned up: ${testFilePath}`, 'test_storage_bucket');
+                }
+                catch (deleteError) {
+                    console.log(`Warning: Could not delete test file ${testFilePath}: ${deleteError.message}`, 'test_storage_bucket');
+                }
+            }
+            else {
+                writeError = 'Test file was not created successfully';
+            }
+        }
+        catch (error) {
+            writeError = error.message;
+            console.log(`Cannot write to bucket: ${error.message}`, 'test_storage_bucket');
+        }
+        // Test 5: Check bucket configuration
+        let bucketConfig = {};
+        try {
+            if (metadata) {
+                bucketConfig = {
+                    location: metadata.location,
+                    storageClass: metadata.storageClass,
+                    timeCreated: metadata.timeCreated,
+                    updated: metadata.updated,
+                    versioning: metadata.versioning?.enabled || false,
+                    cors: metadata.cors ? metadata.cors.length : 0,
+                    lifecycle: metadata.lifecycle ? metadata.lifecycle.rule?.length || 0 : 0
+                };
+            }
+        }
+        catch (configError) {
+            console.log(`Could not retrieve bucket configuration: ${configError.message}`, 'test_storage_bucket');
+        }
+        const isFullyAccessible = exists && metadataAccessible && canListFiles && canWrite;
+        const result = {
+            exists: true,
+            accessible: isFullyAccessible,
+            bucketName: testBucketName,
+            fileCount,
+            configuration: bucketConfig,
+            tests: {
+                existence: exists,
+                metadata: metadataAccessible,
+                listFiles: canListFiles,
+                writeAccess: canWrite
+            },
+            permissions: {
+                read: canListFiles,
+                write: canWrite,
+                admin: metadataAccessible
+            },
+            errors: {
+                write: writeError
+            }
+        };
+        console.log(`Bucket test completed. Fully accessible: ${isFullyAccessible}`, 'test_storage_bucket');
+        console.log(`Test result: ${JSON.stringify(result, null, 2)}`, 'test_storage_bucket');
+        return result;
+    }
+    catch (error) {
+        console.log(`Storage bucket test failed: ${error.message}`, 'test_storage_bucket');
+        return {
+            exists: false,
+            accessible: false,
+            bucketName: bucketName || 'default',
+            error: error.message,
+            tests: {
+                existence: false,
+                metadata: false,
+                listFiles: false,
+                writeAccess: false
+            },
+            permissions: {
+                read: false,
+                write: false,
+                admin: false
+            }
+        };
+    }
+}
+/**
+ * Retry downloadFile with exponential backoff and file validation
+ * @param {string} url - URL to download from
+ * @param {string} filePath - Local path to save the file
+ * @param {Object} options - Retry options
+ * @param {number} options.maxAttempts - Maximum number of retry attempts (default: 10)
+ * @param {number} options.initialDelay - Initial delay in milliseconds (default: 10000)
+ * @param {number} options.maxDelay - Maximum delay in milliseconds (default: 300000)
+ * @param {Function} options.onProgress - Progress callback function
+ * @param {boolean} options.validateFile - Whether to validate the downloaded file (default: true)
+ * @returns {Promise<void>} - Resolves when download succeeds
+ */
+async function downloadFileWithRetry(url, filePath, options = {}) {
+    const { maxAttempts = 15, initialDelay = 10000, // 10 seconds
+    maxDelay = 300000, // 5 minutes
+    onProgress = null, validateFile = true } = options;
+    let lastError;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            if (onProgress) {
+                onProgress(`Download attempt ${attempt}/${maxAttempts}`);
+            }
+            console_log_it(`Download attempt ${attempt}/${maxAttempts} for ${url}`, 'download_file_retry');
+            await downloadFile(url, filePath);
+            // Validate the downloaded file if requested
+            if (validateFile) {
+                try {
+                    console_log_it(`Validating downloaded file: ${filePath}`, 'download_file_retry');
+                    // Check if file exists and has size > 0
+                    const stats = fs.statSync(filePath);
+                    if (stats.size === 0) {
+                        throw new Error('Downloaded file is empty');
+                    }
+                    // Determine file type based on extension and content
+                    const ext = path.extname(filePath).toLowerCase();
+                    const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif'];
+                    const videoExtensions = ['.mp4', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv'];
+                    if (imageExtensions.includes(ext)) {
+                        // Validate image files with Sharp
+                        const metadata = await sharp(filePath).metadata();
+                        console_log_it(`Image validation successful: ${metadata.width}x${metadata.height} ${metadata.format}`, 'download_file_retry');
+                    }
+                    else if (videoExtensions.includes(ext)) {
+                        // For video files, just check if they're readable by ffmpeg (basic validation)
+                        console_log_it(`Video file detected: ${ext}, basic validation passed`, 'download_file_retry');
+                        // Could add ffprobe validation here if needed
+                    }
+                    else {
+                        // For other file types, just verify file exists and has content
+                        console_log_it(`Non-media file detected: ${ext}, size: ${stats.size} bytes`, 'download_file_retry');
+                    }
+                }
+                catch (validationError) {
+                    console_log_it(`File validation failed: ${validationError.message}`, 'download_file_retry');
+                    // Delete the corrupted file
+                    try {
+                        fs.unlinkSync(filePath);
+                        console_log_it(`Deleted corrupted file: ${filePath}`, 'download_file_retry');
+                    }
+                    catch (deleteError) {
+                        console_log_it(`Failed to delete corrupted file: ${deleteError.message}`, 'download_file_retry');
+                    }
+                    throw new Error(`Downloaded file validation failed: ${validationError.message}`);
+                }
+            }
+            console_log_it(`Download and validation successful on attempt ${attempt}`, 'download_file_retry');
+            if (onProgress) {
+                onProgress(`Download completed successfully`);
+            }
+            return; // Success - exit the function
+        }
+        catch (error) {
+            lastError = error;
+            console_log_it(`Download attempt ${attempt} failed: ${error.message}`, 'download_file_retry');
+            if (attempt === maxAttempts) {
+                // Last attempt failed
+                console_log_it(`All ${maxAttempts} download attempts failed`, 'download_file_retry');
+                if (onProgress) {
+                    onProgress(`Download failed after ${maxAttempts} attempts`);
+                }
+                throw new Error(`Download failed after ${maxAttempts} attempts. Last error: ${error.message}`);
+            }
+            // Calculate delay for next attempt (exponential backoff)
+            const delayMs = Math.min(initialDelay * Math.pow(2, attempt - 1), maxDelay);
+            console_log_it(`Waiting ${delayMs}ms before retry attempt ${attempt + 1}`, 'download_file_retry');
+            if (onProgress) {
+                onProgress(`Retrying in ${Math.round(delayMs / 1000)} seconds...`);
+            }
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+        }
+    }
+}
+/**
+ * Uploads a file to Firebase Storage at a specific location
+ * @param {string} localFilePath - Path to the local file to upload
+ * @param {string} bucketPath - Destination path in the bucket (e.g., 'uploads/images/myfile.jpg')
+ * @param {Object} options - Upload options
+ * @param {string} options.contentType - MIME type of the file (auto-detected if not provided)
+ * @param {Object} options.metadata - Additional metadata for the file
+ * @param {boolean} options.public - Whether to make the file publicly accessible (default: false)
+ * @param {Function} options.onProgress - Progress callback function
+ * @returns {Promise<Object>} Upload result with download URL and metadata
+ */
+async function uploadFileToBucket(localFilePath, bucketPath, options = {}) {
+    try {
+        const bucket = getBucket();
+        console.log(`Starting upload: ${localFilePath} -> ${bucketPath}`, 'upload_file_to_bucket');
+        // Normalize bucket path - replace backslashes with forward slashes for GCS compatibility
+        const normalizedBucketPath = bucketPath.replace(/\\/g, '/');
+        console.log(`Normalized bucket path: ${normalizedBucketPath}`, 'upload_file_to_bucket');
+        // Check if local file exists
+        if (!fs.existsSync(localFilePath)) {
+            throw new Error(`Local file does not exist: ${localFilePath}`);
+        }
+        // Get file stats
+        const stats = fs.statSync(localFilePath);
+        const fileSizeMB = (stats.size / 1024 / 1024).toFixed(2);
+        console.log(`File size: ${fileSizeMB} MB`, 'upload_file_to_bucket');
+        // Auto-detect content type if not provided
+        let contentType = options.contentType;
+        if (!contentType) {
+            const ext = path.extname(localFilePath).toLowerCase();
+            const mimeTypes = {
+                '.jpg': 'image/jpeg',
+                '.jpeg': 'image/jpeg',
+                '.png': 'image/png',
+                '.gif': 'image/gif',
+                '.webp': 'image/webp',
+                '.mp4': 'video/mp4',
+                '.mov': 'video/quicktime',
+                '.avi': 'video/x-msvideo',
+                '.json': 'application/json',
+                '.txt': 'text/plain',
+                '.zip': 'application/zip',
+                '.pdf': 'application/pdf'
+            };
+            contentType = mimeTypes[ext] || 'application/octet-stream';
+        }
+        // Prepare upload options
+        const uploadOptions = {
+            destination: normalizedBucketPath,
+            metadata: {
+                contentType,
+                metadata: {
+                    uploadedAt: new Date().toISOString(),
+                    originalName: path.basename(localFilePath),
+                    size: stats.size.toString(),
+                    ...options.metadata
+                }
+            },
+            resumable: stats.size > 5 * 1024 * 1024, // Use resumable upload for files > 5MB
+        };
+        // Set public access if requested
+        if (options.public) {
+            uploadOptions.metadata.metadata.publicAccess = 'true';
+        }
+        // Create file reference
+        const file = bucket.file(normalizedBucketPath);
+        // Upload the file
+        const [uploadedFile] = await bucket.upload(localFilePath, uploadOptions);
+        console.log(`Upload completed: ${normalizedBucketPath}`, 'upload_file_to_bucket');
+        // Make public if requested
+        if (options.public) {
+            await uploadedFile.makePublic();
+            console.log(`File made public: ${normalizedBucketPath}`, 'upload_file_to_bucket');
+        }
+        // Get download URL
+        let downloadURL;
+        if (options.public) {
+            downloadURL = `https://storage.googleapis.com/${bucket.name}/${normalizedBucketPath}`;
+        }
+        else {
+            // Generate signed URL for private files (valid for 1 hour by default)
+            const [url] = await uploadedFile.getSignedUrl({
+                action: 'read',
+                expires: Date.now() + 60 * 60 * 1000, // 1 hour
+            });
+            downloadURL = url;
+        }
+        // Get file metadata
+        const [metadata] = await uploadedFile.getMetadata();
+        const result = {
+            success: true,
+            bucketPath: normalizedBucketPath,
+            downloadURL,
+            contentType,
+            size: stats.size,
+            sizeMB: fileSizeMB,
+            metadata: metadata.metadata,
+            timeCreated: metadata.timeCreated,
+            updated: metadata.updated
+        };
+        console.log(`Upload result: ${JSON.stringify(result, null, 2)}`, 'upload_file_to_bucket');
+        return result;
+    }
+    catch (error) {
+        console.log(`Upload failed: ${error.message}`, 'upload_file_to_bucket');
+        throw new Error(`Failed to upload file to bucket: ${error.message}`);
+    }
+}
+/**
+ * Downloads a file from Firebase Storage bucket to local filesystem
+ * @param {string} bucketPath - Path to the file in the bucket (e.g., 'uploads/images/myfile.jpg')
+ * @param {string} localFilePath - Local path where the file should be saved
+ * @param {Object} options - Download options
+ * @param {Function} options.onProgress - Progress callback function
+ * @param {boolean} options.createDirs - Whether to create directories if they don't exist (default: true)
+ * @param {boolean} options.overwrite - Whether to overwrite existing file (default: true)
+ * @param {boolean} options.validateFile - Whether to validate the downloaded file (default: true)
+ * @returns {Promise<Object>} Download result with file metadata
+ */
+async function downloadFileFromBucket(bucketPath, localFilePath, options = {}) {
+    try {
+        const { onProgress = null, createDirs = true, overwrite = true, validateFile = true } = options;
+        const bucket = getBucket();
+        console.log(`Starting download: ${bucketPath} -> ${localFilePath}`);
+        // Normalize bucket path - replace backslashes with forward slashes for GCS compatibility
+        const normalizedBucketPath = bucketPath.replace(/\\/g, '/');
+        console.log(`Normalized bucket path: ${normalizedBucketPath}`, 'download_file_from_bucket');
+        // Create file reference
+        const file = bucket.file(normalizedBucketPath);
+        // Check if file exists in bucket
+        const [exists] = await file.exists();
+        if (!exists) {
+            throw new Error(`File does not exist in bucket: ${normalizedBucketPath}`);
+        }
+        // Check if local file already exists
+        if (fs.existsSync(localFilePath) && !overwrite) {
+            throw new Error(`Local file already exists and overwrite is disabled: ${localFilePath}`);
+        }
+        // Create local directories if needed
+        if (createDirs) {
+            const localDir = path.dirname(localFilePath);
+            if (!fs.existsSync(localDir)) {
+                fs.mkdirSync(localDir, { recursive: true });
+                console.log(`Created directory: ${localDir}`);
+            }
+        }
+        // Get file metadata
+        const [metadata] = await file.getMetadata();
+        const fileSizeMB = (parseInt(metadata.size) / 1024 / 1024).toFixed(2);
+        console.log(`File size: ${fileSizeMB} MB`, 'download_file_from_bucket');
+        if (onProgress) {
+            await onProgress({
+                type: 'start',
+                bucketPath,
+                localFilePath,
+                size: parseInt(metadata.size),
+                sizeMB: fileSizeMB
+            });
+        }
+        // Download the file
+        await new Promise((resolve, reject) => {
+            const writeStream = fs.createWriteStream(localFilePath);
+            let downloadedBytes = 0;
+            const totalBytes = parseInt(metadata.size);
+            const readStream = file.createReadStream();
+            readStream.on('data', async (chunk) => {
+                downloadedBytes += chunk.length;
+                if (onProgress) {
+                    const percentage = (downloadedBytes / totalBytes) * 100;
+                    await onProgress({
+                        type: 'progress',
+                        downloadedBytes,
+                        totalBytes,
+                        percentage: percentage.toFixed(1)
+                    });
+                }
+            });
+            readStream.on('error', (error) => {
+                writeStream.destroy();
+                if (fs.existsSync(localFilePath)) {
+                    fs.unlinkSync(localFilePath);
+                }
+                reject(error);
+            });
+            writeStream.on('error', (error) => {
+                readStream.destroy();
+                if (fs.existsSync(localFilePath)) {
+                    fs.unlinkSync(localFilePath);
+                }
+                reject(error);
+            });
+            writeStream.on('finish', () => {
+                resolve();
+            });
+            readStream.pipe(writeStream);
+        });
+        console_log_it(`Download completed: ${localFilePath}`, 'download_file_from_bucket');
+        // Validate the downloaded file if requested
+        if (validateFile) {
+            try {
+                const stats = fs.statSync(localFilePath);
+                if (stats.size === 0) {
+                    throw new Error('Downloaded file is empty');
+                }
+                if (stats.size !== parseInt(metadata.size)) {
+                    throw new Error(`File size mismatch. Expected: ${metadata.size}, Actual: ${stats.size}`);
+                }
+                // Additional validation based on file type
+                const ext = path.extname(localFilePath).toLowerCase();
+                const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif'];
+                if (imageExtensions.includes(ext)) {
+                    const imageMetadata = await sharp(localFilePath).metadata();
+                    console_log_it(`Image validation successful: ${imageMetadata.width}x${imageMetadata.height} ${imageMetadata.format}`, 'download_file_from_bucket');
+                }
+            }
+            catch (validationError) {
+                console_log_it(`File validation failed: ${validationError.message}`, 'download_file_from_bucket');
+                // Delete the corrupted file
+                try {
+                    fs.unlinkSync(localFilePath);
+                    console_log_it(`Deleted corrupted file: ${localFilePath}`, 'download_file_from_bucket');
+                }
+                catch (deleteError) {
+                    console_log_it(`Failed to delete corrupted file: ${deleteError.message}`, 'download_file_from_bucket');
+                }
+                throw new Error(`Downloaded file validation failed: ${validationError.message}`);
+            }
+        }
+        if (onProgress) {
+            await onProgress({
+                type: 'complete',
+                bucketPath: normalizedBucketPath,
+                localFilePath,
+                size: parseInt(metadata.size),
+                sizeMB: fileSizeMB
+            });
+        }
+        const result = {
+            success: true,
+            bucketPath: normalizedBucketPath,
+            localFilePath,
+            size: parseInt(metadata.size),
+            sizeMB: fileSizeMB,
+            contentType: metadata.contentType,
+            timeCreated: metadata.timeCreated,
+            updated: metadata.updated,
+            metadata: metadata.metadata
+        };
+        console_log_it(`Download result: ${JSON.stringify(result, null, 2)}`, 'download_file_from_bucket');
+        return result;
+    }
+    catch (error) {
+        console_log_it(`Download failed: ${error.message}`, 'download_file_from_bucket');
+        throw new Error(`Failed to download file from bucket: ${error.message}`);
+    }
+}
+/**
+ * Downloads multiple files from Firebase Storage
+ * @param {Array} fileList - Array of {bucketPath, localPath, options} objects
+ * @param {Function} onProgress - Progress callback function
+ * @returns {Promise<Object>} Download results for all files
+ */
+export async function downloadMultipleFilesFromBucket(fileList, onProgress) {
+    const results = {
+        success: [],
+        failed: [],
+        totalFiles: fileList.length,
+        totalProcessed: 0
+    };
+    for (let i = 0; i < fileList.length; i++) {
+        const { bucketPath, localPath, options = {} } = fileList[i];
+        try {
+            if (onProgress) {
+                onProgress({
+                    type: 'file_start',
+                    current: i + 1,
+                    total: fileList.length,
+                    bucketPath,
+                    localPath
+                });
+            }
+            const result = await downloadFileFromBucket(bucketPath, localPath, {
+                ...options,
+                onProgress: (progressData) => {
+                    if (onProgress) {
+                        onProgress({
+                            type: 'file_progress',
+                            current: i + 1,
+                            total: fileList.length,
+                            bucketPath,
+                            localPath,
+                            progress: progressData
+                        });
+                    }
+                }
+            });
+            results.success.push({ bucketPath, localPath, ...result });
+            results.totalProcessed++;
+            if (onProgress) {
+                onProgress({
+                    type: 'file_complete',
+                    current: i + 1,
+                    total: fileList.length,
+                    bucketPath,
+                    localPath,
+                    result
+                });
+            }
+        }
+        catch (error) {
+            console_log_it(`Failed to download ${bucketPath}: ${error.message}`, 'download_multiple_files');
+            results.failed.push({ bucketPath, localPath, error: error.message });
+            if (onProgress) {
+                onProgress({
+                    type: 'file_error',
+                    current: i + 1,
+                    total: fileList.length,
+                    bucketPath,
+                    localPath,
+                    error: error.message
+                });
+            }
+        }
+    }
+    console_log_it(`Batch download complete. Success: ${results.success.length}, Failed: ${results.failed.length}`, 'download_multiple_files');
+    return results;
+}
+/**
+ * Downloads a file from Firebase Storage with retry logic and exponential backoff
+ * @param {string} bucketPath - Path to the file in the bucket
+ * @param {string} localFilePath - Local path where the file should be saved
+ * @param {Object} options - Download and retry options
+ * @param {number} options.maxAttempts - Maximum number of retry attempts (default: 3)
+ * @param {number} options.initialDelay - Initial delay in milliseconds (default: 1000)
+ * @param {number} options.maxDelay - Maximum delay in milliseconds (default: 30000)
+ * @param {Function} options.onProgress - Progress callback function
+ * @param {boolean} options.createDirs - Whether to create directories if they don't exist (default: true)
+ * @param {boolean} options.overwrite - Whether to overwrite existing file (default: true)
+ * @param {boolean} options.validateFile - Whether to validate the downloaded file (default: true)
+ * @returns {Promise<Object>} Download result with file metadata
+ */
+export async function downloadFileFromBucketWithRetry(bucketPath, localFilePath, options = {}) {
+    const { maxAttempts = 15, initialDelay = 1000, maxDelay = 30000, onProgress = null, ...downloadOptions } = options;
+    let lastError;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            if (onProgress) {
+                await onProgress({
+                    type: 'retry_attempt',
+                    attempt,
+                    maxAttempts,
+                    bucketPath,
+                    localFilePath
+                });
+            }
+            console_log_it(`Download attempt ${attempt}/${maxAttempts} for ${bucketPath}`, 'download_file_bucket_retry');
+            const result = await downloadFileFromBucket(bucketPath, localFilePath, {
+                ...downloadOptions,
+                onProgress: async (progressData) => {
+                    if (onProgress) {
+                        await onProgress({
+                            type: 'download_progress',
+                            attempt,
+                            maxAttempts,
+                            ...progressData
+                        });
+                    }
+                }
+            });
+            console_log_it(`Download successful on attempt ${attempt}`, 'download_file_bucket_retry');
+            if (onProgress) {
+                await onProgress({
+                    type: 'retry_success',
+                    attempt,
+                    maxAttempts,
+                    result
+                });
+            }
+            return result;
+        }
+        catch (error) {
+            lastError = error;
+            console_log_it(`Download attempt ${attempt} failed: ${error.message}`, 'download_file_bucket_retry');
+            if (attempt === maxAttempts) {
+                console_log_it(`All ${maxAttempts} download attempts failed`, 'download_file_bucket_retry');
+                if (onProgress) {
+                    await onProgress({
+                        type: 'retry_failed',
+                        attempt,
+                        maxAttempts,
+                        error: error.message
+                    });
+                }
+                throw new Error(`Download failed after ${maxAttempts} attempts. Last error: ${error.message}`);
+            }
+            // Calculate delay for next attempt (exponential backoff)
+            const delayMs = Math.min(initialDelay * Math.pow(2, attempt - 1), maxDelay);
+            console_log_it(`Waiting ${delayMs}ms before retry attempt ${attempt + 1}`, 'download_file_bucket_retry');
+            if (onProgress) {
+                await onProgress({
+                    type: 'retry_delay',
+                    attempt,
+                    maxAttempts,
+                    delayMs,
+                    error: error.message
+                });
+            }
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+        }
+    }
+}
+/**
+ * Uploads multiple files to Firebase Storage
+ * @param {Array} fileList - Array of {localPath, bucketPath, options} objects
+ * @param {Function} onProgress - Progress callback function
+ * @returns {Promise<Object>} Upload results for all files
+ */
+export async function uploadMultipleFilesToBucket(fileList, onProgress) {
+    const results = {
+        success: [],
+        failed: [],
+        totalFiles: fileList.length,
+        totalProcessed: 0
+    };
+    for (let i = 0; i < fileList.length; i++) {
+        const { localPath, bucketPath, options = {} } = fileList[i];
+        try {
+            if (onProgress) {
+                onProgress({
+                    type: 'file_start',
+                    current: i + 1,
+                    total: fileList.length,
+                    localPath,
+                    bucketPath
+                });
+            }
+            const result = await uploadFileToBucket(localPath, bucketPath, options);
+            results.success.push({ localPath, bucketPath, ...result });
+            results.totalProcessed++;
+            if (onProgress) {
+                onProgress({
+                    type: 'file_complete',
+                    current: i + 1,
+                    total: fileList.length,
+                    localPath,
+                    bucketPath,
+                    result
+                });
+            }
+        }
+        catch (error) {
+            console_log_it(`Failed to upload ${localPath}: ${error.message}`, 'upload_multiple_files');
+            results.failed.push({ localPath, bucketPath, error: error.message });
+            if (onProgress) {
+                onProgress({
+                    type: 'file_error',
+                    current: i + 1,
+                    total: fileList.length,
+                    localPath,
+                    bucketPath,
+                    error: error.message
+                });
+            }
+        }
+    }
+    console_log_it(`Batch upload complete. Success: ${results.success.length}, Failed: ${results.failed.length}`, 'upload_multiple_files');
+    return results;
+}
+/**
+ * Lists files in a Firebase Storage bucket directory
+ * @param {string} prefix - Directory prefix to search (e.g., 'uploads/images/')
+ * @param {Object} options - List options
+ * @param {number} options.maxResults - Maximum number of files to return
+ * @param {string} options.delimiter - Delimiter for directory-like listing
+ * @param {string} options.pageToken - Token for pagination
+ * @returns {Promise<Object>} List of files and metadata
+ */
+export async function listBucketFiles(prefix = '', options = {}) {
+    try {
+        const bucket = getBucket();
+        console_log_it(`Listing files in bucket with prefix: '${prefix}'`, 'list_bucket_files');
+        const listOptions = {
+            prefix,
+            ...options
+        };
+        const [files, , metadata] = await bucket.getFiles(listOptions);
+        const fileList = files.map(file => ({
+            name: file.name,
+            bucket: file.bucket.name,
+            size: parseInt(file.metadata.size || 0),
+            sizeMB: (parseInt(file.metadata.size || 0) / 1024 / 1024).toFixed(2),
+            contentType: file.metadata.contentType,
+            timeCreated: file.metadata.timeCreated,
+            updated: file.metadata.updated,
+            md5Hash: file.metadata.md5Hash,
+            etag: file.metadata.etag
+        }));
+        const result = {
+            files: fileList,
+            totalFiles: fileList.length,
+            prefix,
+            nextPageToken: metadata.nextPageToken || null,
+            totalSize: fileList.reduce((sum, file) => sum + file.size, 0),
+            totalSizeMB: (fileList.reduce((sum, file) => sum + file.size, 0) / 1024 / 1024).toFixed(2)
+        };
+        console_log_it(`Found ${result.totalFiles} files, total size: ${result.totalSizeMB} MB`, 'list_bucket_files');
+        return result;
+    }
+    catch (error) {
+        console_log_it(`Failed to list bucket files: ${error.message}`, 'list_bucket_files');
+        throw new Error(`Failed to list bucket files: ${error.message}`);
+    }
+}
+/**
+ * Checks if a file exists in the Firebase Storage bucket
+ * @param {string} bucketPath - Path to the file in the bucket
+ * @returns {Promise<Object>} Existence check result with metadata
+ */
+export async function checkBucketFileExists(bucketPath) {
+    try {
+        const bucket = getBucket();
+        // Normalize bucket path - replace backslashes with forward slashes for GCS compatibility
+        const normalizedBucketPath = bucketPath.replace(/\\/g, '/');
+        const file = bucket.file(normalizedBucketPath);
+        const [exists] = await file.exists();
+        if (exists) {
+            const [metadata] = await file.getMetadata();
+            return {
+                exists: true,
+                bucketPath: normalizedBucketPath,
+                size: parseInt(metadata.size),
+                sizeMB: (parseInt(metadata.size) / 1024 / 1024).toFixed(2),
+                contentType: metadata.contentType,
+                timeCreated: metadata.timeCreated,
+                updated: metadata.updated,
+                md5Hash: metadata.md5Hash,
+                etag: metadata.etag
+            };
+        }
+        else {
+            return {
+                exists: false,
+                bucketPath: normalizedBucketPath
+            };
+        }
+    }
+    catch (error) {
+        console_log_it(`Failed to check file existence: ${error.message}`, 'check_bucket_file_exists');
+        return {
+            exists: false,
+            bucketPath,
+            error: error.message
+        };
+    }
+}
+/**
+ * Deletes a file from Firebase Storage bucket
+ * @param {string} bucketPath - Path to the file in the bucket
+ * @returns {Promise<Object>} Deletion result
+ */
+export async function deleteBucketFile(bucketPath) {
+    try {
+        const bucket = getBucket();
+        // Normalize bucket path - replace backslashes with forward slashes for GCS compatibility
+        const normalizedBucketPath = bucketPath.replace(/\\/g, '/');
+        const file = bucket.file(normalizedBucketPath);
+        // Check if file exists first
+        const [exists] = await file.exists();
+        if (!exists) {
+            return {
+                success: false,
+                bucketPath: normalizedBucketPath,
+                error: 'File does not exist'
+            };
+        }
+        // Get metadata before deletion
+        const [metadata] = await file.getMetadata();
+        // Delete the file
+        await file.delete();
+        console_log_it(`File deleted from bucket: ${normalizedBucketPath}`, 'delete_bucket_file');
+        return {
+            success: true,
+            bucketPath: normalizedBucketPath,
+            size: parseInt(metadata.size),
+            sizeMB: (parseInt(metadata.size) / 1024 / 1024).toFixed(2),
+            deletedAt: new Date().toISOString()
+        };
+    }
+    catch (error) {
+        console_log_it(`Failed to delete file from bucket: ${error.message}`, 'delete_bucket_file');
+        throw new Error(`Failed to delete file from bucket: ${error.message}`);
+    }
+}
+/**
+ * Gets a signed URL for accessing a private file in Firebase Storage
+ * @param {string} bucketPath - Path to the file in the bucket
+ * @param {Object} options - URL options
+ * @param {string} options.action - Action type ('read', 'write', 'delete', 'resumable')
+ * @param {number} options.expires - Expiration time in milliseconds from now (default: 1 hour)
+ * @param {string} options.contentType - Content type for write operations
+ * @returns {Promise<Object>} Signed URL result
+ */
+export async function getBucketFileSignedUrl(bucketPath, options = {}) {
+    try {
+        const { action = 'read', expires = 60 * 60 * 1000, // 1 hour
+        contentType = null } = options;
+        const bucket = getBucket();
+        // Normalize bucket path - replace backslashes with forward slashes for GCS compatibility
+        const normalizedBucketPath = bucketPath.replace(/\\/g, '/');
+        const file = bucket.file(normalizedBucketPath);
+        // Check if file exists for read operations
+        if (action === 'read') {
+            const [exists] = await file.exists();
+            if (!exists) {
+                throw new Error('File does not exist');
+            }
+        }
+        const signOptions = {
+            action,
+            expires: Date.now() + expires
+        };
+        if (contentType && (action === 'write' || action === 'resumable')) {
+            signOptions.contentType = contentType;
+        }
+        const [url] = await file.getSignedUrl(signOptions);
+        console_log_it(`Generated signed URL for ${action} operation: ${normalizedBucketPath}`, 'get_bucket_file_signed_url');
+        return {
+            success: true,
+            bucketPath: normalizedBucketPath,
+            url,
+            action,
+            expires: new Date(Date.now() + expires).toISOString(),
+            expiresIn: expires
+        };
+    }
+    catch (error) {
+        console_log_it(`Failed to generate signed URL: ${error.message}`, 'get_bucket_file_signed_url');
+        throw new Error(`Failed to generate signed URL: ${error.message}`);
+    }
+}
+export function getJobZipPath(job, fileName) {
+    return path.join(getJobZipFolderPath(job), fileName);
+}
+export function getJobZipFolderPath(job) {
+    return `training/sets/${job.ownerUid}/${job.workflowId}`;
+}
+export function getModelBucketPath(jobId, workflowData, stepId, ownerUid) {
+    return `models/users/${ownerUid}/${jobId}/${workflowData.id}/${stepId}.zip`;
+}
+export function getZipResourcesName(jobId, workflowData, stepId) {
+    return `job_${jobId}_workflow_${workflowData.id}_step_${stepId}.zip`;
+}
+export function getZipModelFileName(jobId, workflowData, stepId) {
+    return `job_${jobId}_workflow_${workflowData.id}_step_${stepId}_model.zip`;
+}
+export function modelFileZipDirName(jobId, stepId) {
+    return `${jobId}-${stepId}`;
+}
+async function extractVideoFrame(videoPath, timestampSeconds, outputPath) {
+    return new Promise((resolve, reject) => {
+        ffmpeg(videoPath)
+            .screenshots({
+            timestamps: [timestampSeconds],
+            filename: path.basename(outputPath),
+            folder: path.dirname(outputPath),
+            // size: '?x?' // Keep original resolution, or specify like '1920x1080'
+        })
+            .on('end', () => {
+            console_log_it(`Frame extracted successfully: ${outputPath}`, 'extract_video_frame');
+            resolve(outputPath);
+        })
+            .on('error', (err) => {
+            console_log_it(`Error extracting frame: ${err.message}`, 'extract_video_frame');
+            reject(err);
+        });
+    });
+}
+// Alternative function with more control over output format:
+async function extractVideoFrameAdvanced(videoPath, timestampSeconds, outputPath, options = {}) {
+    const { width = null, height = null, format = 'png', // png, jpg, jpeg, webp
+    quality = 90 // for jpg/jpeg format
+     } = options;
+    return new Promise((resolve, reject) => {
+        let ffmpegCommand = ffmpeg(videoPath)
+            .seekInput(timestampSeconds)
+            .frames(1)
+            .format(format)
+            .output(outputPath);
+        // Set size if specified
+        if (width && height) {
+            ffmpegCommand = ffmpegCommand.size(`${width}x${height}`);
+        }
+        else if (width || height) {
+            ffmpegCommand = ffmpegCommand.size(`${width || '?'}x${height || '?'}`);
+        }
+        // Set quality for JPEG
+        if (format === 'jpg' || format === 'jpeg') {
+            ffmpegCommand = ffmpegCommand.outputOptions([`-q:v ${Math.round((100 - quality) / 10)}`]);
+        }
+        ffmpegCommand
+            .on('end', () => {
+            console_log_it(`Frame extracted successfully: ${outputPath}`, 'extract_video_frame_advanced');
+            resolve(outputPath);
+        })
+            .on('error', (err) => {
+            console_log_it(`Error extracting frame: ${err.message}`, 'extract_video_frame_advanced');
+            reject(err);
+        })
+            .run();
+    });
+}
+/**
+ * Resize an image to fit within maximum dimensions while maintaining aspect ratio
+ * No padding/blank parts will be added - image will be smaller than or equal to max dimensions
+ * @param {string} inputPath - Path to the input image
+ * @param {string} outputPath - Path where the resized image will be saved
+ * @param {Object} dimensions - Maximum dimensions {width, height}
+ * @param {Object} options - Additional options for resizing
+ * @returns {Promise<string>} - Path to the resized image
+ */
+async function resizeImage(inputPath, outputPath, dimensions, options = {}) {
+    const { quality = 90, // JPEG quality (if output is JPEG)
+    format = null, // Auto-detect from output path extension
+    withoutEnlargement = true // Don't enlarge if source is smaller than target
+     } = options;
+    try {
+        console_log_it(`Resizing image from ${inputPath} to ${outputPath}`, 'resize_image');
+        console_log_it(`Maximum dimensions: ${dimensions.width}x${dimensions.height}`, 'resize_image');
+        // Create sharp instance
+        let sharpInstance = sharp(inputPath);
+        // Get original image metadata
+        const metadata = await sharpInstance.metadata();
+        console_log_it(`Original dimensions: ${metadata.width}x${metadata.height}`, 'resize_image');
+        // Calculate aspect ratios
+        const originalAspectRatio = metadata.width / metadata.height;
+        const maxAspectRatio = dimensions.width / dimensions.height;
+        console_log_it(`Original aspect ratio: ${originalAspectRatio.toFixed(3)}`, 'resize_image');
+        console_log_it(`Max container aspect ratio: ${maxAspectRatio.toFixed(3)}`, 'resize_image');
+        // Calculate the actual dimensions to fit within the maximum bounds
+        let newWidth, newHeight;
+        if (originalAspectRatio > maxAspectRatio) {
+            // Image is wider relative to container - constrain by width
+            newWidth = Math.min(dimensions.width, metadata.width);
+            newHeight = Math.round(newWidth / originalAspectRatio);
+        }
+        else {
+            // Image is taller relative to container - constrain by height
+            newHeight = Math.min(dimensions.height, metadata.height);
+            newWidth = Math.round(newHeight * originalAspectRatio);
+        }
+        // If withoutEnlargement is true, don't make image larger than original
+        if (withoutEnlargement) {
+            newWidth = Math.min(newWidth, metadata.width);
+            newHeight = Math.min(newHeight, metadata.height);
+        }
+        console_log_it(`Calculated new dimensions: ${newWidth}x${newHeight}`, 'resize_image');
+        // Resize to exact calculated dimensions (no padding)
+        sharpInstance = sharpInstance.resize({
+            width: newWidth,
+            height: newHeight,
+            fit: 'fill', // Use 'fill' since we calculated exact dimensions
+            withoutEnlargement: false // We already handled enlargement above
+        });
+        // Determine output format
+        let outputFormat = format;
+        if (!outputFormat) {
+            const ext = path.extname(outputPath).toLowerCase();
+            switch (ext) {
+                case '.jpg':
+                case '.jpeg':
+                    outputFormat = 'jpeg';
+                    break;
+                case '.png':
+                    outputFormat = 'png';
+                    break;
+                case '.webp':
+                    outputFormat = 'webp';
+                    break;
+                case '.tiff':
+                case '.tif':
+                    outputFormat = 'tiff';
+                    break;
+                default:
+                    outputFormat = 'png'; // Default fallback
+            }
+        }
+        // Apply format-specific options
+        switch (outputFormat) {
+            case 'jpeg':
+                sharpInstance = sharpInstance.jpeg({ quality: quality });
+                break;
+            case 'png':
+                sharpInstance = sharpInstance.png({ compressionLevel: 6 });
+                break;
+            case 'webp':
+                sharpInstance = sharpInstance.webp({ quality: quality });
+                break;
+            case 'tiff':
+                sharpInstance = sharpInstance.tiff({ quality: quality });
+                break;
+        }
+        // Save the resized image
+        await sharpInstance.toFile(outputPath);
+        // Get final image metadata to confirm dimensions
+        const finalMetadata = await sharp(outputPath).metadata();
+        console_log_it(`Final dimensions: ${finalMetadata.width}x${finalMetadata.height}`, 'resize_image');
+        console_log_it(`Image resized successfully: ${outputPath}`, 'resize_image');
+        return outputPath;
+    }
+    catch (error) {
+        console_log_it(`Error resizing image: ${error.message}`, 'resize_image');
+        throw new Error(`Failed to resize image: ${error.message}`);
+    }
+}
+/**
+ * Get image dimensions using Sharp
+ * @param {string} imagePath - Path to the image file
+ * @returns {Promise<{width: number, height: number}>} - Image dimensions
+ */
+async function getImageSizeWithSharp(imagePath) {
+    try {
+        const metadata = await sharp(imagePath).metadata();
+        if (!metadata.width || !metadata.height) {
+            throw new Error(`Could not determine image dimensions for: ${imagePath}`);
+        }
+        console_log_it(`Image dimensions for ${imagePath}: ${metadata.width}x${metadata.height}`, 'get_image_size_sharp');
+        return {
+            width: metadata.width,
+            height: metadata.height,
+            format: metadata.format,
+            channels: metadata.channels,
+            density: metadata.density
+        };
+    }
+    catch (error) {
+        console_log_it(`Error getting image size for ${imagePath}: ${error.message}`, 'get_image_size_sharp');
+        throw new Error(`Failed to get image dimensions: ${error.message}`);
+    }
+}
+/**
+ * Resize all images in a folder to fit within maximum dimensions (replaces originals)
+ * @param {string} folder - Path to folder containing images to resize
+ * @param {Object} dimensions - Maximum dimensions {width, height}
+ * @param {Object} options - Additional options for resizing
+ * @returns {Promise<Object>} - Results summary with success/failure counts
+ */
+async function resizeImagesInFolder(folder, dimensions, options = {}) {
+    const { quality = 99, format = null, withoutEnlargement = false, // Don't enlarge if source is smaller than target
+    supportedFormats = ['.jpg', '.jpeg', '.png', '.webp', '.tiff', '.tif', '.bmp'], onProgress = null, // Callback function for progress updates
+    onError = null, // Callback function for individual file errors
+    backupOriginals = false, // Create backup copies before resizing
+    backupSuffix = '_original', force_resize = true } = options;
+    try {
+        console_log_it(`Starting in-place resize for folder: ${folder}`, 'resize_images_folder_path');
+        console_log_it(`Target dimensions: ${dimensions.width}x${dimensions.height}`, 'resize_images_folder_dim');
+        // Ensure folder exists
+        if (!fs.existsSync(folder)) {
+            throw new Error(`Folder does not exist: ${folder}`);
+        }
+        // Get all files in the folder
+        const allFiles = fs.readdirSync(folder);
+        // Filter for image files
+        const imageFiles = allFiles.filter(file => {
+            const ext = path.extname(file).toLowerCase();
+            return supportedFormats.includes(ext);
+        });
+        console_log_it(`Found ${imageFiles.length} image files to process`, 'resize_images_folder_count');
+        if (imageFiles.length === 0) {
+            return {
+                success: true,
+                message: 'No image files found to resize',
+                processed: 0,
+                skipped: 0,
+                failed: 0,
+                results: []
+            };
+        }
+        const results = {
+            success: true,
+            processed: 0,
+            skipped: 0,
+            failed: 0,
+            results: []
+        };
+        // Process each image file
+        for (let i = 0; i < imageFiles.length; i++) {
+            const fileName = imageFiles[i];
+            const filePath = path.join(folder, fileName);
+            try {
+                console_log_it(`Processing ${fileName} (${i + 1}/${imageFiles.length})`, 'resize_images_folder');
+                // Get original image dimensions
+                const originalSize = await getImageSizeWithSharp(filePath);
+                console_log_it(`Original size: ${originalSize.width}x${originalSize.height}`, 'resize_images_folder');
+                // Check if image needs resizing
+                if (!force_resize && (originalSize.width <= dimensions.width && originalSize.height <= dimensions.height) && withoutEnlargement) {
+                    console_log_it(`Skipping ${fileName} - already within target dimensions`, 'resize_images_folder');
+                    results.skipped++;
+                    results.results.push({
+                        fileName,
+                        status: 'skipped',
+                        reason: 'Already within target dimensions',
+                        filePath,
+                        originalSize: `${originalSize.width}x${originalSize.height}`
+                    });
+                    // Call progress callback
+                    if (onProgress) {
+                        await onProgress({
+                            current: i + 1,
+                            total: imageFiles.length,
+                            percentage: Math.round(((i + 1) / imageFiles.length) * 100),
+                            fileName,
+                            status: 'skipped',
+                            progress: Math.round(((i + 1) / imageFiles.length) * 100)
+                        });
+                    }
+                    continue;
+                }
+                // Create backup if requested
+                if (backupOriginals) {
+                    const nameWithoutExt = path.parse(fileName).name;
+                    const ext = path.parse(fileName).ext;
+                    const backupFileName = `${nameWithoutExt}${backupSuffix}${ext}`;
+                    const backupPath = path.join(folder, backupFileName);
+                    console_log_it(`Creating backup: ${backupFileName}`, 'resize_images_folder');
+                    fs.copyFileSync(filePath, backupPath);
+                }
+                // Create temporary file for resizing
+                const tempPath = path.join(folder, `temp_resize_${fileName}`);
+                // Resize the image to temporary file
+                await resizeImage(filePath, tempPath, dimensions, {
+                    quality,
+                    format,
+                    withoutEnlargement
+                });
+                // Get resized image dimensions
+                const finalSize = await getImageSizeWithSharp(tempPath);
+                console_log_it(`Resized to: ${finalSize.width}x${finalSize.height}`, 'resize_images_folder');
+                // Replace original with resized image
+                fs.renameSync(tempPath, filePath);
+                results.processed++;
+                results.results.push({
+                    fileName,
+                    status: 'success',
+                    filePath,
+                    originalSize: `${originalSize.width}x${originalSize.height}`,
+                    finalSize: `${finalSize.width}x${finalSize.height}`,
+                    compressionRatio: ((originalSize.width * originalSize.height) / (finalSize.width * finalSize.height)).toFixed(2),
+                    backupCreated: backupOriginals
+                });
+                // Call progress callback
+                if (onProgress) {
+                    await onProgress({
+                        current: i + 1,
+                        total: imageFiles.length,
+                        percentage: Math.round(((i + 1) / imageFiles.length) * 100),
+                        fileName,
+                        status: 'success',
+                        progress: Math.round(((i + 1) / imageFiles.length) * 100),
+                        originalSize,
+                        finalSize
+                    });
+                }
+            }
+            catch (error) {
+                console_log_it(`Error processing ${fileName}: ${error.message}`, 'resize_images_folder');
+                // Clean up temporary file if it exists
+                const tempPath = path.join(folder, `temp_resize_${fileName}`);
+                if (fs.existsSync(tempPath)) {
+                    try {
+                        fs.unlinkSync(tempPath);
+                    }
+                    catch (cleanupError) {
+                        console_log_it(`Warning: Could not clean up temp file ${tempPath}`, 'resize_images_folder');
+                    }
+                }
+                results.failed++;
+                results.results.push({
+                    fileName,
+                    status: 'failed',
+                    reason: error.message,
+                    filePath
+                });
+                // Call error callback
+                if (onError) {
+                    onError({
+                        fileName,
+                        error: error.message,
+                        filePath
+                    });
+                }
+                // Call progress callback for failed items
+                if (onProgress) {
+                    await onProgress({
+                        current: i + 1,
+                        total: imageFiles.length,
+                        percentage: Math.round(((i + 1) / imageFiles.length) * 100),
+                        fileName,
+                        status: 'failed',
+                        progress: Math.round(((i + 1) / imageFiles.length) * 100),
+                        error: error.message
+                    });
+                }
+            }
+        }
+        // Final summary
+        console_log_it(`In-place resize complete. Processed: ${results.processed}, Skipped: ${results.skipped}, Failed: ${results.failed}`, 'resize_images_folder');
+        return results;
+    }
+    catch (error) {
+        console_log_it(`Error in batch resize: ${error.message}`, 'resize_images_folder');
+        throw new Error(`Failed to resize images in folder: ${error.message}`);
+    }
+}
+/**
+ * Recursively resize images in a folder and all subfolders
+ * @param {string} rootFolder - Path to root folder
+ * @param {Object} dimensions - Maximum dimensions {width, height}
+ * @param {Object} options - Additional options
+ * @returns {Promise<Object>} - Combined results from all folders
+ */
+async function resizeImagesRecursive(rootFolder, dimensions, options = {}) {
+    const { onProgress = null, folderFilter = null // Function to filter which folders to process
+     } = options;
+    try {
+        console_log_it(`Starting recursive resize from: ${rootFolder}`, 'resize_images_recursive_start');
+        const allResults = {
+            success: true,
+            totalProcessed: 0,
+            totalSkipped: 0,
+            totalFailed: 0,
+            folderResults: []
+        };
+        // Get all subdirectories
+        const entries = await fs.promises.readdir(rootFolder, { withFileTypes: true, recursive: true });
+        const folders = [...entries.map(t => t.path)]; // Include root folder
+        console_log_it(`Found ${folders.length} folders to process`, 'resize_images_recursive_length');
+        // Process each folder
+        for (let i = 0; i < folders.length; i++) {
+            const folderPath = folders[i];
+            const folderName = path.relative(rootFolder, folderPath) || path.basename(rootFolder);
+            console_log_it(`Processing folder ${i + 1}/${folders.length}: ${folderName}`, 'resize_images_recursive_p');
+            if (onProgress) {
+                await onProgress({
+                    type: 'folder_start',
+                    folder: folderName,
+                    current: i + 1,
+                    total: folders.length,
+                    percentage: Math.round(((i + 1) / folders.length) * 100),
+                    message: `Starting folder: ${folderName}`
+                });
+            }
+            try {
+                const folderResults = await resizeImagesInFolder(folderPath, dimensions, {
+                    ...options,
+                    onProgress: async (fileProgress) => {
+                        if (onProgress) {
+                            await onProgress({
+                                type: 'file_progress',
+                                folder: folderName,
+                                ...fileProgress
+                            });
+                        }
+                    }
+                });
+                allResults.totalProcessed += folderResults.processed;
+                allResults.totalSkipped += folderResults.skipped;
+                allResults.totalFailed += folderResults.failed;
+                allResults.folderResults.push({
+                    folder: folderName,
+                    path: folderPath,
+                    ...folderResults
+                });
+                if (onProgress) {
+                    onProgress({
+                        type: 'folder_complete',
+                        folder: folderName,
+                        ...folderResults
+                    });
+                }
+            }
+            catch (error) {
+                console_log_it(`Error processing folder ${folderName}: ${error.message}`, 'resize_images_recursive');
+                allResults.folderResults.push({
+                    folder: folderName,
+                    path: folderPath,
+                    success: false,
+                    error: error.message
+                });
+            }
+        }
+        console_log_it(`Recursive resize complete. Total processed: ${allResults.totalProcessed}, skipped: ${allResults.totalSkipped}, failed: ${allResults.totalFailed}`, 'resize_images_recursive');
+        return allResults;
+    }
+    catch (error) {
+        console_log_it(`Error in recursive resize: ${error.message}`, 'resize_images_recursive');
+        throw new Error(`Failed to resize images recursively: ${error.message}`);
+    }
+}
+//# sourceMappingURL=capture_training_data_endpoints.js.map