bulltrackers-module 1.0.710 → 1.0.713

package/functions/maintenance/backfill-task-engine-data/index.js

@@ -0,0 +1,844 @@
+/**
+ * @fileoverview Backfill Task Engine Data from Firestore to BigQuery
+ * 
+ * This function reads existing portfolio, trade history, and social post data
+ * from Firestore and writes it to BigQuery tables.
+ * 
+ * Usage (Cloud Function HTTP trigger):
+ *   ?startDate=2024-01-01&endDate=2024-12-31&dataType=all
+ *   ?startDate=2024-01-01&endDate=2024-12-31&dataType=portfolio
+ *   ?startDate=2024-01-01&endDate=2024-12-31&dataType=history
+ *   ?startDate=2024-01-01&endDate=2024-12-31&dataType=social
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31 --dataType=all
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31 --dataType=portfolio
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31 --dataType=history
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31 --dataType=social
+ * 
+ * Features:
+ *   - Processes in batches to avoid timeouts
+ *   - Tracks progress in Firestore (resume capability)
+ *   - Handles both new and legacy Firestore structures
+ *   - Does NOT delete any Firestore data
+ *   - Works both as Cloud Function and local script
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const pLimit = require('p-limit');
+const { 
+    ensurePortfolioSnapshotsTable,
+    ensureTradeHistorySnapshotsTable,
+    ensureSocialPostSnapshotsTable,
+    insertRows,
+    insertRowsWithMerge,
+    checkExistingRows,
+    removeDuplicates
+} = require('../../core/utils/bigquery_utils');
+
+// Reusable timestamp converter (created once, reused)
+const timestampReplacer = (key, value) => {
+    if (value && typeof value === 'object' && value.toDate && typeof value.toDate === 'function') {
+        return value.toDate().toISOString();
+    }
+    return value;
+};
+
+// Firestore batch read size (Firestore getAll limit is 100)
+const FIRESTORE_BATCH_SIZE = 100;
+
+const db = new Firestore();
+const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+const PROGRESS_COLLECTION = 'backfill_progress';
+
+/**
+ * Get all dates between start and end (inclusive)
+ */
+function getDateRange(startDateStr, endDateStr) {
+    const dates = [];
+    const start = new Date(startDateStr);
+    const end = new Date(endDateStr);
+    
+    for (let d = new Date(start); d <= end; d.setUTCDate(d.getUTCDate() + 1)) {
+        dates.push(d.toISOString().split('T')[0]);
+    }
+    
+    return dates;
+}
+
+/**
+ * Check if a date has already been backfilled
+ */
+async function isDateBackfilled(dataType, dateStr) {
+    const progressRef = db.collection(PROGRESS_COLLECTION)
+        .doc(dataType)
+        .collection('dates')
+        .doc(dateStr);
+    
+    const snap = await progressRef.get();
+    return snap.exists && snap.data().completed === true;
+}
+
+/**
+ * Mark a date as backfilled
+ * @param {string} dataType - Data type ('portfolio', 'history', 'social')
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {number} rowCount - Number of rows successfully inserted
+ * @param {number} expectedCount - Expected total rows (for verification)
+ */
+async function markDateBackfilled(dataType, dateStr, rowCount, expectedCount = null) {
+    const progressRef = db.collection(PROGRESS_COLLECTION)
+        .doc(dataType)
+        .collection('dates')
+        .doc(dateStr);
+    
+    await progressRef.set({
+        completed: true,
+        rowCount: rowCount,
+        expectedCount: expectedCount || rowCount, // Store expected for verification
+        completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+    }, { merge: true });
+}
+
+/**
+ * Backfill portfolio data for a single date
+ */
+async function backfillPortfolioData(dateStr, logger) {
+    const rows = [];
+    
+    try {
+        // Collect all document references first
+        const allDocRefs = [];
+        
+        // NEW STRUCTURE: PopularInvestorPortfolioData/{date}/{cid}/{cid}
+        const piPortCollection = db.collection('PopularInvestorPortfolioData').doc(dateStr);
+        const piSubcollections = await piPortCollection.listCollections();
+        for (const subcol of piSubcollections) {
+            const cid = subcol.id;
+            allDocRefs.push({ ref: subcol.doc(cid), type: 'POPULAR_INVESTOR', cid });
+        }
+        
+        // NEW STRUCTURE: SignedInUserPortfolioData/{date}/{cid}/{cid}
+        const signedInPortCollection = db.collection('SignedInUserPortfolioData').doc(dateStr);
+        const signedInSubcollections = await signedInPortCollection.listCollections();
+        for (const subcol of signedInSubcollections) {
+            const cid = subcol.id;
+            allDocRefs.push({ ref: subcol.doc(cid), type: 'SIGNED_IN_USER', cid });
+        }
+        
+        if (allDocRefs.length === 0) {
+            logger.log('INFO', `[Backfill] Portfolio: No data found for ${dateStr}, skipping`);
+            return 0;
+        }
+        
+        // Batch read all documents using getAll() (up to 100 per call, much faster)
+        for (let i = 0; i < allDocRefs.length; i += FIRESTORE_BATCH_SIZE) {
+            const batch = allDocRefs.slice(i, i + FIRESTORE_BATCH_SIZE);
+            const docRefs = batch.map(item => item.ref);
+            
+            // Single RPC call for up to 100 documents
+            const docs = await db.getAll(...docRefs);
+            
+            for (let j = 0; j < docs.length; j++) {
+                const doc = docs[j];
+                if (doc.exists) {
+                    const data = doc.data();
+                    const batchItem = batch[j];
+                    
+                    // Convert Firestore Timestamps to ISO strings (reusable function)
+                    const cleanData = JSON.parse(JSON.stringify(data, timestampReplacer));
+                    
+                    rows.push({
+                        date: dateStr,
+                        user_id: Number(batchItem.cid) || 0,
+                        user_type: batchItem.type,
+                        portfolio_data: JSON.stringify(cleanData), // BigQuery JSON type requires a string
+                        fetched_at: data.fetchedAt?.toDate?.()?.toISOString() || new Date().toISOString()
+                    });
+                }
+            }
+        }
+        
+        // LEGACY STRUCTURE: NormalUserPortfolios/{blockId}/snapshots/{date}/parts/{partId}
+        // Note: Legacy structure is more complex, would need to read parts and merge
+        // For now, we'll skip legacy and focus on new structure
+        // TODO: Add legacy structure support if needed
+        
+        if (rows.length > 0) {
+            await ensurePortfolioSnapshotsTable(logger);
+            
+            // Use BigQuery MERGE for efficient deduplication (SQL-native, most efficient)
+            // MERGE handles duplicates natively - no need to check first
+            if (process.env.USE_MERGE_FOR_DEDUP === 'true') {
+                logger.log('INFO', `[Backfill] Portfolio: Using MERGE for ${dateStr} (${rows.length} rows, SQL-native deduplication)`);
+                try {
+                    const rowsInserted = await insertRowsWithMerge(
+                        datasetId, 
+                        'portfolio_snapshots', 
+                        rows, 
+                        ['date', 'user_id', 'user_type'], // Unique key fields
+                        logger
+                    );
+                    logger.log('INFO', `[Backfill] Portfolio: MERGE completed for ${dateStr} - ${rowsInserted} new rows inserted`);
+                    return rowsInserted;
+                } catch (mergeError) {
+                    logger.log('WARN', `[Backfill] Portfolio MERGE failed for ${dateStr}, falling back to check-then-insert: ${mergeError.message}`);
+                    // Fall through to check-then-insert method
+                }
+            }
+            
+            // Fallback: Check for existing rows to prevent duplicates
+            // Can skip if SKIP_DUPLICATE_CHECK=true (for fresh backfills)
+            let newRows = rows;
+            if (process.env.SKIP_DUPLICATE_CHECK !== 'true') {
+                const existingKeys = await checkExistingRows(datasetId, 'portfolio_snapshots', dateStr, rows, logger);
+                
+                // Filter out rows that already exist
+                newRows = rows.filter(row => {
+                    const key = `${row.user_id}|${row.user_type}`;
+                    return !existingKeys.has(key);
+                });
+                
+                if (newRows.length < rows.length) {
+                    logger.log('INFO', `[Backfill] Portfolio: Skipping ${rows.length - newRows.length} duplicate rows for ${dateStr}`);
+                }
+            } else {
+                logger.log('INFO', `[Backfill] Portfolio: Skipping duplicate check for ${dateStr} (SKIP_DUPLICATE_CHECK=true)`);
+            }
+            
+            if (newRows.length > 0) {
+                // Insert in smaller batches to avoid request size limits
+                const BATCH_SIZE = 50; // Insert 50 rows at a time
+                let totalInserted = 0;
+                const failedBatches = [];
+                for (let i = 0; i < newRows.length; i += BATCH_SIZE) {
+                    const batch = newRows.slice(i, i + BATCH_SIZE);
+                    try {
+                        await insertRows(datasetId, 'portfolio_snapshots', batch, logger);
+                        totalInserted += batch.length;
+                    } catch (batchError) {
+                        const batchNum = Math.floor(i/BATCH_SIZE) + 1;
+                        logger.log('WARN', `[Backfill] Portfolio batch insert failed for ${dateStr}, batch ${batchNum}: ${batchError.message}`);
+                        failedBatches.push({ batchNum, startIndex: i, batchSize: batch.length });
+                        // Continue with next batch
+                    }
+                }
+                logger.log('INFO', `[Backfill] Portfolio: Inserted ${totalInserted}/${newRows.length} new rows for ${dateStr} (${rows.length - newRows.length} duplicates skipped)`);
+                
+                // Return actual inserted count, and throw if partial failure
+                if (totalInserted < newRows.length) {
+                    const missing = newRows.length - totalInserted;
+                    throw new Error(`Partial failure: Only inserted ${totalInserted}/${newRows.length} rows. ${missing} rows failed. Failed batches: ${failedBatches.map(b => `batch ${b.batchNum}`).join(', ')}`);
+                }
+                
+                return totalInserted + (rows.length - newRows.length); // Return total (new + existing)
+            } else {
+                logger.log('INFO', `[Backfill] Portfolio: All rows already exist for ${dateStr}, skipping`);
+                return rows.length; // All were duplicates, return count found
+            }
+        } else {
+            logger.log('INFO', `[Backfill] Portfolio: No data found for ${dateStr}, skipping`);
+            return 0;
+        }
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Portfolio error for ${dateStr}:`, {
+            message: error.message,
+            code: error.code,
+            errors: error.errors,
+            stack: error.stack
+        });
+        throw error;
+    }
+}
+
+/**
+ * Backfill trade history data for a single date
+ */
+async function backfillHistoryData(dateStr, logger) {
+    const rows = [];
+    
+    try {
+        // Collect all document references first
+        const allDocRefs = [];
+        
+        // NEW STRUCTURE: PopularInvestorTradeHistoryData/{date}/{cid}/{cid}
+        const piHistCollection = db.collection('PopularInvestorTradeHistoryData').doc(dateStr);
+        const piSubcollections = await piHistCollection.listCollections();
+        for (const subcol of piSubcollections) {
+            const cid = subcol.id;
+            allDocRefs.push({ ref: subcol.doc(cid), type: 'POPULAR_INVESTOR', cid });
+        }
+        
+        // NEW STRUCTURE: SignedInUserTradeHistoryData/{date}/{cid}/{cid}
+        const signedInHistCollection = db.collection('SignedInUserTradeHistoryData').doc(dateStr);
+        const signedInSubcollections = await signedInHistCollection.listCollections();
+        for (const subcol of signedInSubcollections) {
+            const cid = subcol.id;
+            allDocRefs.push({ ref: subcol.doc(cid), type: 'SIGNED_IN_USER', cid });
+        }
+        
+        if (allDocRefs.length === 0) {
+            logger.log('INFO', `[Backfill] History: No data found for ${dateStr}, skipping`);
+            return 0;
+        }
+        
+        // Batch read all documents using getAll() (up to 100 per call, much faster)
+        for (let i = 0; i < allDocRefs.length; i += FIRESTORE_BATCH_SIZE) {
+            const batch = allDocRefs.slice(i, i + FIRESTORE_BATCH_SIZE);
+            const docRefs = batch.map(item => item.ref);
+            
+            // Single RPC call for up to 100 documents
+            const docs = await db.getAll(...docRefs);
+            
+            for (let j = 0; j < docs.length; j++) {
+                const doc = docs[j];
+                if (doc.exists) {
+                    const data = doc.data();
+                    const batchItem = batch[j];
+                    
+                    // Convert Firestore Timestamps to ISO strings (reusable function)
+                    const cleanData = JSON.parse(JSON.stringify(data, timestampReplacer));
+                    
+                    rows.push({
+                        date: dateStr,
+                        user_id: Number(batchItem.cid) || 0,
+                        user_type: batchItem.type,
+                        history_data: JSON.stringify(cleanData), // BigQuery JSON type requires a string
+                        fetched_at: data.fetchedAt?.toDate?.()?.toISOString() || new Date().toISOString()
+                    });
+                }
+            }
+        }
+        
+        if (rows.length > 0) {
+            await ensureTradeHistorySnapshotsTable(logger);
+            
+            // Use BigQuery MERGE for efficient deduplication (SQL-native, most efficient)
+            if (process.env.USE_MERGE_FOR_DEDUP === 'true') {
+                logger.log('INFO', `[Backfill] History: Using MERGE for ${dateStr} (${rows.length} rows, SQL-native deduplication)`);
+                try {
+                    const rowsInserted = await insertRowsWithMerge(
+                        datasetId, 
+                        'trade_history_snapshots', 
+                        rows, 
+                        ['date', 'user_id', 'user_type'], // Unique key fields
+                        logger
+                    );
+                    logger.log('INFO', `[Backfill] History: MERGE completed for ${dateStr} - ${rowsInserted} new rows inserted`);
+                    return rowsInserted;
+                } catch (mergeError) {
+                    logger.log('WARN', `[Backfill] History MERGE failed for ${dateStr}, falling back to check-then-insert: ${mergeError.message}`);
+                    // Fall through to check-then-insert method
+                }
+            }
+            
+            // Fallback: Check for existing rows to prevent duplicates
+            let newRows = rows;
+            if (process.env.SKIP_DUPLICATE_CHECK !== 'true') {
+                const existingKeys = await checkExistingRows(datasetId, 'trade_history_snapshots', dateStr, rows, logger);
+                
+                // Filter out rows that already exist
+                newRows = rows.filter(row => {
+                    const key = `${row.user_id}|${row.user_type}`;
+                    return !existingKeys.has(key);
+                });
+                
+                if (newRows.length < rows.length) {
+                    logger.log('INFO', `[Backfill] History: Skipping ${rows.length - newRows.length} duplicate rows for ${dateStr}`);
+                }
+            } else {
+                logger.log('INFO', `[Backfill] History: Skipping duplicate check for ${dateStr} (SKIP_DUPLICATE_CHECK=true)`);
+            }
+            
+            if (newRows.length > 0) {
+                // Insert in smaller batches to avoid request size limits (trade history can be very large)
+                const BATCH_SIZE = 25; // Smaller batches for history (can be very large)
+                let totalInserted = 0;
+                const failedBatches = [];
+                for (let i = 0; i < newRows.length; i += BATCH_SIZE) {
+                    const batch = newRows.slice(i, i + BATCH_SIZE);
+                    try {
+                        await insertRows(datasetId, 'trade_history_snapshots', batch, logger);
+                        totalInserted += batch.length;
+                    } catch (batchError) {
+                        const batchNum = Math.floor(i/BATCH_SIZE) + 1;
+                        logger.log('WARN', `[Backfill] History batch insert failed for ${dateStr}, batch ${batchNum}: ${batchError.message}`);
+                        failedBatches.push({ batchNum, startIndex: i, batchSize: batch.length });
+                        // Continue with next batch
+                    }
+                }
+                logger.log('INFO', `[Backfill] History: Inserted ${totalInserted}/${newRows.length} new rows for ${dateStr} (${rows.length - newRows.length} duplicates skipped)`);
+                
+                // Return actual inserted count, and throw if partial failure
+                if (totalInserted < newRows.length) {
+                    const missing = newRows.length - totalInserted;
+                    throw new Error(`Partial failure: Only inserted ${totalInserted}/${newRows.length} rows. ${missing} rows failed. Failed batches: ${failedBatches.map(b => `batch ${b.batchNum}`).join(', ')}`);
+                }
+                
+                return totalInserted + (rows.length - newRows.length); // Return total (new + existing)
+            } else {
+                logger.log('INFO', `[Backfill] History: All rows already exist for ${dateStr}, skipping`);
+                return rows.length; // All were duplicates, return count found
+            }
+        } else {
+            logger.log('INFO', `[Backfill] History: No data found for ${dateStr}, skipping`);
+            return 0;
+        }
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] History error for ${dateStr}:`, {
+            message: error.message,
+            code: error.code,
+            errors: error.errors,
+            stack: error.stack
+        });
+        throw error;
+    }
+}
+
+/**
+ * Backfill social post data for a single date
+ */
+async function backfillSocialData(dateStr, logger) {
+    const rows = [];
+    
+    try {
+        // Collect all document references first
+        const allDocRefs = [];
+        
+        // NEW STRUCTURE: PopularInvestorSocialPostData/{date}/{cid}/{cid}
+        const piSocialCollection = db.collection('PopularInvestorSocialPostData').doc(dateStr);
+        const piSubcollections = await piSocialCollection.listCollections();
+        for (const subcol of piSubcollections) {
+            const cid = subcol.id;
+            allDocRefs.push({ ref: subcol.doc(cid), type: 'POPULAR_INVESTOR', cid });
+        }
+        
+        // NEW STRUCTURE: SignedInUserSocialPostData/{date}/{cid}/{cid}
+        const signedInSocialCollection = db.collection('SignedInUserSocialPostData').doc(dateStr);
+        const signedInSubcollections = await signedInSocialCollection.listCollections();
+        for (const subcol of signedInSubcollections) {
+            const cid = subcol.id;
+            allDocRefs.push({ ref: subcol.doc(cid), type: 'SIGNED_IN_USER', cid });
+        }
+        
+        if (allDocRefs.length === 0) {
+            logger.log('INFO', `[Backfill] Social: No data found for ${dateStr}, skipping`);
+            return 0;
+        }
+        
+        // Batch read all documents using getAll() (up to 100 per call, much faster)
+        for (let i = 0; i < allDocRefs.length; i += FIRESTORE_BATCH_SIZE) {
+            const batch = allDocRefs.slice(i, i + FIRESTORE_BATCH_SIZE);
+            const docRefs = batch.map(item => item.ref);
+            
+            // Single RPC call for up to 100 documents
+            const docs = await db.getAll(...docRefs);
+            
+            for (let j = 0; j < docs.length; j++) {
+                const doc = docs[j];
+                if (doc.exists) {
+                    const data = doc.data();
+                    const batchItem = batch[j];
+                    
+                    // Convert Firestore Timestamps to ISO strings (reusable function)
+                    const cleanData = JSON.parse(JSON.stringify(data, timestampReplacer));
+                    
+                    rows.push({
+                        date: dateStr,
+                        user_id: Number(batchItem.cid) || 0,
+                        user_type: batchItem.type,
+                        posts_data: JSON.stringify(cleanData), // BigQuery JSON type requires a string
+                        fetched_at: data.fetchedAt?.toDate?.()?.toISOString() || new Date().toISOString()
+                    });
+                }
+            }
+        }
+        
+        if (rows.length > 0) {
+            await ensureSocialPostSnapshotsTable(logger);
+            
+            // Use BigQuery MERGE for efficient deduplication (SQL-native, most efficient)
+            if (process.env.USE_MERGE_FOR_DEDUP === 'true') {
+                logger.log('INFO', `[Backfill] Social: Using MERGE for ${dateStr} (${rows.length} rows, SQL-native deduplication)`);
+                try {
+                    const rowsInserted = await insertRowsWithMerge(
+                        datasetId, 
+                        'social_post_snapshots', 
+                        rows, 
+                        ['date', 'user_id', 'user_type'], // Unique key fields
+                        logger
+                    );
+                    logger.log('INFO', `[Backfill] Social: MERGE completed for ${dateStr} - ${rowsInserted} new rows inserted`);
+                    return rowsInserted;
+                } catch (mergeError) {
+                    logger.log('WARN', `[Backfill] Social MERGE failed for ${dateStr}, falling back to check-then-insert: ${mergeError.message}`);
+                    // Fall through to check-then-insert method
+                }
+            }
+            
+            // Fallback: Check for existing rows to prevent duplicates
+            let newRows = rows;
+            if (process.env.SKIP_DUPLICATE_CHECK !== 'true') {
+                const existingKeys = await checkExistingRows(datasetId, 'social_post_snapshots', dateStr, rows, logger);
+                
+                // Filter out rows that already exist
+                newRows = rows.filter(row => {
+                    const key = `${row.user_id}|${row.user_type}`;
+                    return !existingKeys.has(key);
+                });
+                
+                if (newRows.length < rows.length) {
+                    logger.log('INFO', `[Backfill] Social: Skipping ${rows.length - newRows.length} duplicate rows for ${dateStr}`);
+                }
+            } else {
+                logger.log('INFO', `[Backfill] Social: Skipping duplicate check for ${dateStr} (SKIP_DUPLICATE_CHECK=true)`);
+            }
+            
+            if (newRows.length > 0) {
+                // Insert in smaller batches to avoid request size limits
+                const BATCH_SIZE = 50; // Insert 50 rows at a time
+                let totalInserted = 0;
+                const failedBatches = [];
+                for (let i = 0; i < newRows.length; i += BATCH_SIZE) {
+                    const batch = newRows.slice(i, i + BATCH_SIZE);
+                    try {
+                        await insertRows(datasetId, 'social_post_snapshots', batch, logger);
+                        totalInserted += batch.length;
+                    } catch (batchError) {
+                        const batchNum = Math.floor(i/BATCH_SIZE) + 1;
+                        logger.log('WARN', `[Backfill] Social batch insert failed for ${dateStr}, batch ${batchNum}: ${batchError.message}`);
+                        failedBatches.push({ batchNum, startIndex: i, batchSize: batch.length });
+                        // Continue with next batch
+                    }
+                }
+                logger.log('INFO', `[Backfill] Social: Inserted ${totalInserted}/${newRows.length} new rows for ${dateStr} (${rows.length - newRows.length} duplicates skipped)`);
+                
+                // Return actual inserted count, and throw if partial failure
+                if (totalInserted < newRows.length) {
+                    const missing = newRows.length - totalInserted;
+                    throw new Error(`Partial failure: Only inserted ${totalInserted}/${newRows.length} rows. ${missing} rows failed. Failed batches: ${failedBatches.map(b => `batch ${b.batchNum}`).join(', ')}`);
+                }
+                
+                return totalInserted + (rows.length - newRows.length); // Return total (new + existing)
+            } else {
+                logger.log('INFO', `[Backfill] Social: All rows already exist for ${dateStr}, skipping`);
+                return rows.length; // All were duplicates, return count found
+            }
+        } else {
+            logger.log('INFO', `[Backfill] Social: No data found for ${dateStr}, skipping`);
+            return 0;
+        }
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Social error for ${dateStr}:`, {
+            message: error.message,
+            code: error.code,
+            errors: error.errors,
+            stack: error.stack
+        });
+        throw error;
+    }
+}
+
+/**
+ * Main backfill function
+ * Works both as Cloud Function (req/res) and local script (no req/res)
+ */
+async function backfillTaskEngineData(req, res) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    // Determine if running as Cloud Function or local script
+    const isCloudFunction = req && res;
+    
+    // Extract parameters from either HTTP request or command line
+    let startDate, endDate, dataType;
+    
+    if (isCloudFunction) {
+        // Cloud Function: get from query params
+        startDate = req.query.startDate || '2024-01-01';
+        endDate = req.query.endDate || new Date().toISOString().split('T')[0];
+        dataType = req.query.dataType || 'all';
+    } else {
+        // Local script: get from command line args
+        const args = process.argv.slice(2);
+        const parseArg = (key, defaultValue) => {
+            const arg = args.find(a => a.startsWith(`--${key}=`));
+            return arg ? arg.split('=')[1] : defaultValue;
+        };
+        
+        startDate = parseArg('startDate', '2024-01-01');
+        endDate = parseArg('endDate', new Date().toISOString().split('T')[0]);
+        dataType = parseArg('dataType', 'all');
+    }
+    
+    try {
+        logger.log('INFO', `[Backfill] Starting backfill: ${dataType} from ${startDate} to ${endDate}`);
+        logger.log('INFO', `[Backfill] Running in ${isCloudFunction ? 'Cloud Function' : 'local'} mode`);
+        
+        const dates = getDateRange(startDate, endDate);
+        logger.log('INFO', `[Backfill] Processing ${dates.length} dates`);
+        
+        const summary = {
+            totalDates: dates.length,
+            processedDates: 0,
+            skippedDates: 0,
+            portfolioRows: 0,
+            historyRows: 0,
+            socialRows: 0,
+            errors: []
+        };
+        
+        // Process dates in parallel with concurrency limit
+        // Limit to 5 concurrent dates to avoid overwhelming BigQuery/Firestore
+        const dateLimit = pLimit(5);
+        
+        // Helper function to retry operations with exponential backoff
+        async function retryWithBackoff(fn, maxRetries = 3, baseDelay = 1000) {
+            for (let attempt = 1; attempt <= maxRetries; attempt++) {
+                try {
+                    return await fn();
+                } catch (error) {
+                    // Check if error is retryable (timeout, rate limit, etc.)
+                    const isRetryable = error.code === 4 || // DEADLINE_EXCEEDED (timeout)
+                                      error.code === 8 || // RESOURCE_EXHAUSTED (rate limit)
+                                      error.code === 14 || // UNAVAILABLE (service unavailable)
+                                      error.message?.includes('timeout') ||
+                                      error.message?.includes('rate limit') ||
+                                      error.message?.includes('exceeded');
+                    
+                    if (!isRetryable || attempt === maxRetries) {
+                        throw error; // Not retryable or out of retries
+                    }
+                    
+                    const delay = baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+                    logger.log('WARN', `[Backfill] Retryable error (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms: ${error.message}`);
+                    await new Promise(resolve => setTimeout(resolve, delay));
+                }
+            }
+        }
+        
+        // Process all dates in parallel (with limit)
+        const datePromises = dates.map(dateStr => 
+            dateLimit(async () => {
+                const dateResults = { date: dateStr, portfolio: null, history: null, social: null };
+                
+                // Process each data type independently - if one fails, others can still succeed (self-healing)
+                // Process data types in parallel for better performance
+                const dataTypeTasks = [];
+                
+                logger.log('INFO', `[Backfill] Processing date ${dateStr} with dataType=${dataType}`);
+                
+                // Portfolio
+                if (dataType === 'all' || dataType === 'portfolio') {
+                    logger.log('INFO', `[Backfill] Adding portfolio task for ${dateStr}`);
+                    dataTypeTasks.push((async () => {
+                        try {
+                            logger.log('INFO', `[Backfill] Starting portfolio backfill for ${dateStr}`);
+                            if (!(await isDateBackfilled('portfolio', dateStr))) {
+                                const count = await retryWithBackoff(
+                                    () => backfillPortfolioData(dateStr, logger),
+                                    3, // 3 retries
+                                    2000 // 2s base delay
+                                );
+                                summary.portfolioRows += count;
+                                // Only mark as backfilled if we actually inserted data (count > 0)
+                                if (count > 0) {
+                                    await markDateBackfilled('portfolio', dateStr, count);
+                                }
+                                dateResults.portfolio = { success: true, count };
+                                logger.log('INFO', `[Backfill] ✅ Portfolio completed for ${dateStr}: ${count} rows`);
+                            } else {
+                                logger.log('INFO', `[Backfill] Portfolio for ${dateStr} already backfilled, skipping`);
+                                summary.skippedDates++;
+                                dateResults.portfolio = { success: true, skipped: true };
+                            }
+                        } catch (portfolioError) {
+                            const errorDetails = {
+                                message: portfolioError.message,
+                                code: portfolioError.code,
+                                errors: portfolioError.errors
+                            };
+                            logger.log('ERROR', `[Backfill] Portfolio failed for ${dateStr} (after retries):`, errorDetails);
+                            summary.errors.push({ date: dateStr, dataType: 'portfolio', error: portfolioError.message });
+                            dateResults.portfolio = { success: false, error: portfolioError.message };
+                        }
+                    })());
+                } else {
+                    logger.log('INFO', `[Backfill] Skipping portfolio for ${dateStr} (dataType=${dataType})`);
+                }
+                
+                // History
+                if (dataType === 'all' || dataType === 'history') {
+                    logger.log('INFO', `[Backfill] Adding history task for ${dateStr}`);
+                    dataTypeTasks.push((async () => {
+                        try {
+                            logger.log('INFO', `[Backfill] Starting history backfill for ${dateStr}`);
+                            if (!(await isDateBackfilled('history', dateStr))) {
+                                const count = await retryWithBackoff(
+                                    () => backfillHistoryData(dateStr, logger),
+                                    3, // 3 retries
+                                    2000 // 2s base delay
+                                );
+                                summary.historyRows += count;
+                                if (count > 0) {
+                                    await markDateBackfilled('history', dateStr, count);
+                                }
+                                dateResults.history = { success: true, count };
+                                logger.log('INFO', `[Backfill] ✅ History completed for ${dateStr}: ${count} rows`);
+                            } else {
+                                logger.log('INFO', `[Backfill] History for ${dateStr} already backfilled, skipping`);
+                                dateResults.history = { success: true, skipped: true };
+                            }
+                        } catch (historyError) {
+                            const errorDetails = {
+                                message: historyError.message,
+                                code: historyError.code,
+                                errors: historyError.errors
+                            };
+                            logger.log('ERROR', `[Backfill] History failed for ${dateStr} (after retries):`, errorDetails);
+                            summary.errors.push({ date: dateStr, dataType: 'history', error: historyError.message });
+                            dateResults.history = { success: false, error: historyError.message };
+                        }
+                    })());
+                } else {
+                    logger.log('INFO', `[Backfill] Skipping history for ${dateStr} (dataType=${dataType})`);
+                }
+                
+                // Social
+                if (dataType === 'all' || dataType === 'social') {
+                    logger.log('INFO', `[Backfill] Adding social task for ${dateStr}`);
+                    dataTypeTasks.push((async () => {
+                        try {
+                            logger.log('INFO', `[Backfill] Starting social backfill for ${dateStr}`);
+                            if (!(await isDateBackfilled('social', dateStr))) {
+                                const count = await retryWithBackoff(
+                                    () => backfillSocialData(dateStr, logger),
+                                    3, // 3 retries
+                                    2000 // 2s base delay
+                                );
+                                summary.socialRows += count;
+                                if (count > 0) {
+                                    await markDateBackfilled('social', dateStr, count);
+                                }
+                                dateResults.social = { success: true, count };
+                                logger.log('INFO', `[Backfill] ✅ Social completed for ${dateStr}: ${count} rows`);
+                            } else {
+                                logger.log('INFO', `[Backfill] Social for ${dateStr} already backfilled, skipping`);
+                                dateResults.social = { success: true, skipped: true };
+                            }
+                        } catch (socialError) {
+                            const errorDetails = {
+                                message: socialError.message,
+                                code: socialError.code,
+                                errors: socialError.errors
+                            };
+                            logger.log('ERROR', `[Backfill] Social failed for ${dateStr} (after retries):`, errorDetails);
+                            summary.errors.push({ date: dateStr, dataType: 'social', error: socialError.message });
+                            dateResults.social = { success: false, error: socialError.message };
+                        }
+                    })());
+                } else {
+                    logger.log('INFO', `[Backfill] Skipping social for ${dateStr} (dataType=${dataType})`);
+                }
+                
+                logger.log('INFO', `[Backfill] Executing ${dataTypeTasks.length} data type tasks in parallel for ${dateStr}`);
+                
+                // Wait for all data types to complete in parallel
+                await Promise.all(dataTypeTasks);
+                
+                logger.log('INFO', `[Backfill] All data type tasks completed for ${dateStr}`);
+                
+                // Date is considered processed even if some data types failed
+                // Failed data types will be retried on next run (self-healing)
+                summary.processedDates++;
+                const allSuccess = Object.values(dateResults).every(r => r === null || r.success);
+                return { date: dateStr, success: allSuccess, results: dateResults };
+            })
+        );
+        
+        // Wait for all dates to complete, logging progress periodically
+        logger.log('INFO', `[Backfill] Processing ${dates.length} dates in parallel (max 5 concurrent)`);
+        const results = await Promise.all(datePromises);
+        
+        const successful = results.filter(r => r.success).length;
+        const failed = results.filter(r => !r.success).length;
+        logger.log('INFO', `[Backfill] Completed: ${successful} successful, ${failed} failed`);
+        
+        // Optional: Remove duplicates at the end (cheaper than checking each date)
+        // Only run if DEDUPLICATE_AT_END=true and we processed all data types
+        if (process.env.DEDUPLICATE_AT_END === 'true' && (dataType === 'all' || dataType === 'portfolio' || dataType === 'history' || dataType === 'social')) {
+            logger.log('INFO', `[Backfill] Running final deduplication pass...`);
+            
+            const dedupTasks = [];
+            if (dataType === 'all' || dataType === 'portfolio') {
+                dedupTasks.push(
+                    removeDuplicates(datasetId, 'portfolio_snapshots', 'date', ['user_id', 'user_type'], logger)
+                        .then(count => logger.log('INFO', `[Backfill] Deduplication: Removed ${count} duplicate portfolio rows`))
+                        .catch(err => logger.log('WARN', `[Backfill] Portfolio deduplication failed: ${err.message}`))
+                );
+            }
+            if (dataType === 'all' || dataType === 'history') {
+                dedupTasks.push(
+                    removeDuplicates(datasetId, 'trade_history_snapshots', 'date', ['user_id', 'user_type'], logger)
+                        .then(count => logger.log('INFO', `[Backfill] Deduplication: Removed ${count} duplicate history rows`))
+                        .catch(err => logger.log('WARN', `[Backfill] History deduplication failed: ${err.message}`))
+                );
+            }
+            if (dataType === 'all' || dataType === 'social') {
+                dedupTasks.push(
+                    removeDuplicates(datasetId, 'social_post_snapshots', 'date', ['user_id', 'user_type'], logger)
+                        .then(count => logger.log('INFO', `[Backfill] Deduplication: Removed ${count} duplicate social rows`))
+                        .catch(err => logger.log('WARN', `[Backfill] Social deduplication failed: ${err.message}`))
+                );
+            }
+            
+            await Promise.all(dedupTasks);
+            logger.log('INFO', `[Backfill] Deduplication complete`);
+        }
+        
+        logger.log('INFO', `[Backfill] Complete! Summary:`, JSON.stringify(summary, null, 2));
+        
+        if (isCloudFunction) {
+            res.status(200).json({
+                success: true,
+                message: 'Backfill completed',
+                summary: summary
+            });
+        } else {
+            // Local script: just log and exit
+            console.log('\n✅ Backfill completed successfully!');
+            process.exit(0);
+        }
+        
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`, error);
+        
+        if (isCloudFunction) {
+            res.status(500).json({
+                success: false,
+                error: error.message
+            });
+        } else {
+            console.error('\n❌ Backfill failed:', error.message);
+            process.exit(1);
+        }
+    }
+}
+
+// If running as a script directly (not imported as module), execute immediately
+if (require.main === module) {
+    // Running as local script
+    console.log('🚀 Starting local backfill script...\n');
+    console.log('Usage: node index.js --startDate=YYYY-MM-DD --endDate=YYYY-MM-DD --dataType=all|portfolio|history|social\n');
+    
+    backfillTaskEngineData(null, null).catch(error => {
+        console.error('Fatal error:', error);
+        process.exit(1);
+    });
+}
+
+module.exports = { backfillTaskEngineData };