bulltrackers-module 1.0.719 → 1.0.720

package/functions/maintenance/backfill-pi-alert-history/index.js

@@ -0,0 +1,201 @@
+/**
+ * @fileoverview Backfill PI Alert History from Firestore to BigQuery
+ * 
+ * This function reads existing PI alert history data from Firestore (PIAlertHistoryData/{date})
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ *   node index.js (backfills all dates)
+ * 
+ * Features:
+ *   - Backfills alert history for date range (or all dates)
+ *   - Uses load jobs (free) for efficient batching
+ *   - Handles compressed Firestore data
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensurePIAlertHistoryTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill PI alert history from Firestore to BigQuery for a date range
+ */
+async function backfillAlertHistory(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting PI alert history backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensurePIAlertHistoryTable(logger);
+        
+        const alertHistoryCollection = db.collection('PIAlertHistoryData');
+        
+        // Get all alert history documents
+        logger.log('INFO', '[Backfill] Fetching alert history documents from Firestore...');
+        const snapshot = await alertHistoryCollection.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No alert history documents found in Firestore');
+            return { success: false, message: 'No alert history found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} alert history documents`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each document
+        for (const doc of snapshot.docs) {
+            const dateStr = doc.id; // Document ID is the date (YYYY-MM-DD)
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                const data = tryDecompress(doc.data());
+                const { date, lastUpdated, ...piAlertHistory } = data;
+                
+                if (!piAlertHistory || Object.keys(piAlertHistory).length === 0) {
+                    logger.log('WARN', `[Backfill] No alert history in document for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Transform to BigQuery rows
+                const lastUpdatedTimestamp = lastUpdated 
+                    ? (lastUpdated.toDate ? lastUpdated.toDate().toISOString() : lastUpdated)
+                    : new Date().toISOString();
+                
+                const bigqueryRows = [];
+                for (const [piId, alertData] of Object.entries(piAlertHistory)) {
+                    if (alertData && typeof alertData === 'object') {
+                        // Each PI can have multiple alert types
+                        for (const [alertType, alertInfo] of Object.entries(alertData)) {
+                            if (alertInfo && typeof alertInfo === 'object') {
+                                const lastTriggered = alertInfo.lastTriggered
+                                    ? (alertInfo.lastTriggered.toDate ? alertInfo.lastTriggered.toDate().toISOString() : alertInfo.lastTriggered)
+                                    : null;
+                                
+                                bigqueryRows.push({
+                                    date: dateStr,
+                                    pi_id: parseInt(piId, 10),
+                                    alert_type: alertType,
+                                    triggered: alertInfo.triggered || false,
+                                    trigger_count: alertInfo.count || 0,
+                                    triggered_for: alertInfo.triggeredFor || [],
+                                    metadata: alertInfo.metadata || {},
+                                    last_triggered: lastTriggered,
+                                    last_updated: lastUpdatedTimestamp
+                                });
+                            }
+                        }
+                    }
+                }
+                
+                if (bigqueryRows.length === 0) {
+                    logger.log('WARN', `[Backfill] No valid alert history rows for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'pi_alert_history', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process alert history for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ PI alert history backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] PI alert history backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillPIAlertHistory(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting PI Alert History backfill...');
+    
+    try {
+        const result = await backfillAlertHistory(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI execution
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillPIAlertHistory(startDate, endDate)
+        .then(result => {
+            console.log('Backfill result:', result);
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillPIAlertHistory };

package/functions/maintenance/backfill-pi-page-views/index.js

@@ -0,0 +1,189 @@
+/**
+ * @fileoverview Backfill PI Page Views from Firestore to BigQuery
+ * 
+ * This function reads existing PI page views data from Firestore (PIPageViewsData/{date})
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ *   node index.js (backfills all dates)
+ * 
+ * Features:
+ *   - Backfills page views for date range (or all dates)
+ *   - Uses load jobs (free) for efficient batching
+ *   - Handles compressed Firestore data
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensurePIPageViewsTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill PI page views from Firestore to BigQuery for a date range
+ */
+async function backfillPageViews(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting PI page views backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensurePIPageViewsTable(logger);
+        
+        const pageViewsCollection = db.collection('PIPageViewsData');
+        
+        // Get all page views documents
+        logger.log('INFO', '[Backfill] Fetching page views documents from Firestore...');
+        const snapshot = await pageViewsCollection.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No page views documents found in Firestore');
+            return { success: false, message: 'No page views found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} page views documents`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each document
+        for (const doc of snapshot.docs) {
+            const dateStr = doc.id; // Document ID is the date (YYYY-MM-DD)
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                const data = tryDecompress(doc.data());
+                const { date, lastUpdated, ...piPageViews } = data;
+                
+                if (!piPageViews || Object.keys(piPageViews).length === 0) {
+                    logger.log('WARN', `[Backfill] No page views in document for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Transform to BigQuery rows
+                const lastUpdatedTimestamp = lastUpdated 
+                    ? (lastUpdated.toDate ? lastUpdated.toDate().toISOString() : lastUpdated)
+                    : new Date().toISOString();
+                
+                const bigqueryRows = [];
+                for (const [piId, pageViewData] of Object.entries(piPageViews)) {
+                    if (pageViewData && typeof pageViewData === 'object') {
+                        bigqueryRows.push({
+                            date: dateStr,
+                            pi_id: parseInt(piId, 10),
+                            total_views: pageViewData.totalViews || null,
+                            unique_viewers: pageViewData.uniqueViewers || null,
+                            views_by_user: pageViewData.viewsByUser || {},
+                            last_updated: lastUpdatedTimestamp
+                        });
+                    }
+                }
+                
+                if (bigqueryRows.length === 0) {
+                    logger.log('WARN', `[Backfill] No valid page views rows for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'pi_page_views', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process page views for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ PI page views backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] PI page views backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillPIPageViews(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting PI Page Views backfill...');
+    
+    try {
+        const result = await backfillPageViews(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI execution
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillPIPageViews(startDate, endDate)
+        .then(result => {
+            console.log('Backfill result:', result);
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillPIPageViews };

package/functions/maintenance/backfill-pi-ratings/index.js

@@ -0,0 +1,376 @@
+/**
+ * @fileoverview Backfill PI Ratings from Firestore to BigQuery
+ * 
+ * This function reads existing PI ratings data from Firestore (PiReviews/{date}/shards/)
+ * aggregates them by PI, and writes to BigQuery table.
+ * 
+ * Note: Raw reviews are stored in PiReviews/{date}/shards/, not in PIRatingsData/{date}
+ * This script aggregates the raw reviews into the expected format.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ *   node index.js (backfills all dates)
+ * 
+ * Features:
+ *   - Backfills ratings for date range (or all dates)
+ *   - Aggregates raw reviews from shards
+ *   - Uses load jobs (free) for efficient batching
+ *   - Handles compressed Firestore data
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensurePIRatingsTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill PI ratings from Firestore to BigQuery for a date range
+ * Reads from PiReviews/{date}/shards/ and aggregates by PI
+ */
+async function backfillRatings(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting PI ratings backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensurePIRatingsTable(logger);
+        
+        // Use collection group query to find all shards
+        // Path structure: PiReviews/{date}/shards/{shardId}
+        logger.log('INFO', '[Backfill] Fetching all shards from PiReviews using collection group query...');
+        const shardsCollectionGroup = db.collectionGroup('shards');
+        const allShardsSnapshot = await shardsCollectionGroup.get();
+        
+        if (allShardsSnapshot.empty) {
+            logger.log('WARN', '[Backfill] No shards found in collection group query');
+            return { success: false, message: 'No shards found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${allShardsSnapshot.size} shard documents total`);
+        
+        // Group shards by date (extract from parent path: PiReviews/{date}/shards/{shardId})
+        // Filter to only shards under PiReviews collection
+        const shardsByDate = {};
+        let samplePath = null;
+        
+        for (const shardDoc of allShardsSnapshot.docs) {
+            // Path structure: PiReviews/{date}/shards/{shardId}
+            // Get full path: projects/{project}/databases/{db}/documents/PiReviews/{date}/shards/{shardId}
+            const fullPath = shardDoc.ref.path;
+            
+            // Log first path for debugging
+            if (!samplePath) {
+                samplePath = fullPath;
+                logger.log('INFO', `[Backfill] Sample shard path: ${fullPath}`);
+            }
+            
+            // Check if this shard is under PiReviews collection
+            if (!fullPath.includes('PiReviews')) {
+                continue; // Skip shards from other collections
+            }
+            
+            // Extract date from path: PiReviews/{date}/shards/{shardId}
+            // Path format: projects/{project}/databases/{db}/documents/PiReviews/{date}/shards/{shardId}
+            // Or simpler: PiReviews/{date}/shards/{shardId}
+            const pathParts = fullPath.split('/');
+            
+            // Find PiReviews in path
+            let piReviewsIndex = -1;
+            for (let i = 0; i < pathParts.length; i++) {
+                if (pathParts[i] === 'PiReviews') {
+                    piReviewsIndex = i;
+                    break;
+                }
+            }
+            
+            if (piReviewsIndex >= 0 && pathParts[piReviewsIndex + 1]) {
+                const dateStr = pathParts[piReviewsIndex + 1];
+                
+                // Validate date format (YYYY-MM-DD)
+                if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) {
+                    if (!shardsByDate[dateStr]) {
+                        shardsByDate[dateStr] = [];
+                    }
+                    shardsByDate[dateStr].push(shardDoc);
+                } else {
+                    logger.log('DEBUG', `[Backfill] Skipping invalid date format: ${dateStr} from path ${fullPath}`);
+                }
+            } else {
+                logger.log('DEBUG', `[Backfill] Could not extract date from path: ${fullPath}`);
+            }
+        }
+        
+        const uniqueDates = Object.keys(shardsByDate);
+        logger.log('INFO', `[Backfill] Found shards for ${uniqueDates.length} unique dates: ${uniqueDates.slice(0, 5).join(', ')}${uniqueDates.length > 5 ? '...' : ''}`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each date
+        for (const dateStr of uniqueDates) {
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                // Get all shards for this date (already grouped)
+                const shardsForDate = shardsByDate[dateStr];
+                
+                if (!shardsForDate || shardsForDate.length === 0) {
+                    logger.log('WARN', `[Backfill] No shards found for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Aggregate reviews by PI
+                const reviewsByPi = {};
+                let totalReviewsFound = 0;
+                let skippedReviews = 0;
+                
+                // Helper to convert Firestore timestamp to ISO string
+                function timestampToISO(timestamp) {
+                    if (!timestamp) return null;
+                    if (timestamp.toDate) {
+                        return timestamp.toDate().toISOString();
+                    } else if (timestamp._seconds) {
+                        return new Date(timestamp._seconds * 1000).toISOString();
+                    } else if (timestamp instanceof Date) {
+                        return timestamp.toISOString();
+                    }
+                    return null;
+                }
+                
+                for (const shardDoc of shardsForDate) {
+                    const shardData = tryDecompress(shardDoc.data());
+                    
+                    if (!shardData || typeof shardData !== 'object') {
+                        continue;
+                    }
+                    
+                    // Shard data structure: Keys are like "reviews.29312236_31075566" with review data as values
+                    // Process all keys that contain review data
+                    Object.keys(shardData).forEach(key => {
+                        const entry = shardData[key];
+                        totalReviewsFound++;
+                        
+                        // Skip if not an object
+                        if (!entry || typeof entry !== 'object') {
+                            skippedReviews++;
+                            return;
+                        }
+                        
+                        // Check for required fields (piCid and rating)
+                        if (entry.piCid === undefined || entry.rating === undefined) {
+                            skippedReviews++;
+                            return;
+                        }
+                        
+                        const piId = String(entry.piCid);
+                        
+                        if (!reviewsByPi[piId]) {
+                            reviewsByPi[piId] = {
+                                allReviews: [], // Store all review objects with full metadata
+                                ratingsByUser: {},
+                                userTimestamps: {} // Track timestamps to keep most recent rating per user for aggregation
+                            };
+                        }
+                        
+                        // Store full review metadata
+                        const reviewMetadata = {
+                            action: entry.action || null,
+                            actualUserCid: entry.actualUserCid || null,
+                            comment: entry.comment || null,
+                            createdAt: timestampToISO(entry.createdAt),
+                            isAnonymous: entry.isAnonymous || false,
+                            isImpersonating: entry.isImpersonating || false,
+                            loggedAt: timestampToISO(entry.loggedAt),
+                            piCid: entry.piCid,
+                            rating: entry.rating,
+                            reviewId: entry.reviewId || null,
+                            reviewerUsername: entry.reviewerUsername || null,
+                            updatedAt: timestampToISO(entry.updatedAt),
+                            userCid: entry.userCid || null
+                        };
+                        
+                        reviewsByPi[piId].allReviews.push(reviewMetadata);
+                        
+                        // Also track for aggregation (use most recent rating per user)
+                        // Add rating (handle legacy format where rating might be 0-50 instead of 1-5)
+                        let rating = entry.rating;
+                        if (rating > 5) {
+                            rating = Math.round(rating / 10);
+                        }
+                        // Ensure rating is between 1-5
+                        rating = Math.max(1, Math.min(5, rating));
+                        
+                        // Track rating by user (use the most recent rating if user has multiple reviews/edits)
+                        const userId = String(entry.userCid || entry.actualUserCid || '');
+                        
+                        if (userId && userId !== 'undefined' && userId !== 'null') {
+                            // Get timestamp for comparison (use updatedAt if available, else createdAt, else loggedAt)
+                            let entryTimestamp = 0;
+                            const timestampField = entry.updatedAt || entry.createdAt || entry.loggedAt;
+                            
+                            if (timestampField) {
+                                if (timestampField.toDate) {
+                                    entryTimestamp = timestampField.toDate().getTime();
+                                } else if (timestampField._seconds) {
+                                    entryTimestamp = timestampField._seconds * 1000;
+                                } else if (timestampField instanceof Date) {
+                                    entryTimestamp = timestampField.getTime();
+                                }
+                            }
+                            
+                            const existingTimestamp = reviewsByPi[piId].userTimestamps[userId] || 0;
+                            
+                            // Only keep if this is a new review or more recent than existing
+                            if (entryTimestamp >= existingTimestamp) {
+                                reviewsByPi[piId].ratingsByUser[userId] = rating;
+                                reviewsByPi[piId].userTimestamps[userId] = entryTimestamp;
+                            }
+                        }
+                    });
+                }
+                
+                if (totalReviewsFound > 0) {
+                    logger.log('INFO', `[Backfill] Processed ${totalReviewsFound} review entries for ${dateStr} (${skippedReviews} skipped, ${totalReviewsFound - skippedReviews} valid)`);
+                }
+                
+                if (Object.keys(reviewsByPi).length === 0) {
+                    logger.log('WARN', `[Backfill] No reviews found in shards for ${dateStr} (found ${totalReviewsFound} entries, ${skippedReviews} skipped)`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                logger.log('INFO', `[Backfill] Aggregated reviews for ${Object.keys(reviewsByPi).length} PIs for ${dateStr}`);
+                
+                // Transform to BigQuery rows
+                const lastUpdatedTimestamp = new Date().toISOString();
+                const bigqueryRows = [];
+                
+                for (const [piId, piData] of Object.entries(reviewsByPi)) {
+                    // Use unique user ratings (one rating per user - most recent if edited)
+                    const uniqueRatings = Object.values(piData.ratingsByUser);
+                    const totalRatings = uniqueRatings.length; // Count of unique users who rated
+                    const averageRating = totalRatings > 0 
+                        ? uniqueRatings.reduce((sum, r) => sum + r, 0) / totalRatings 
+                        : null;
+                    
+                    bigqueryRows.push({
+                        date: dateStr,
+                        pi_id: parseInt(piId, 10),
+                        average_rating: averageRating ? Number(averageRating.toFixed(2)) : null,
+                        total_ratings: totalRatings,
+                        ratings_by_user: piData.ratingsByUser || {},
+                        reviews: piData.allReviews || [], // Full review metadata array
+                        last_updated: lastUpdatedTimestamp
+                    });
+                }
+                
+                logger.log('INFO', `[Backfill] Created ${bigqueryRows.length} BigQuery rows for ${dateStr}`);
+                
+                if (bigqueryRows.length === 0) {
+                    logger.log('WARN', `[Backfill] No valid ratings rows for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'pi_ratings', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process ratings for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ PI ratings backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] PI ratings backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillPIRatings(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting PI Ratings backfill...');
+    
+    try {
+        const result = await backfillRatings(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI execution
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillPIRatings(startDate, endDate)
+        .then(result => {
+            console.log('Backfill result:', result);
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillPIRatings };

package/functions/maintenance/backfill-price-data-from-firestore/index.js

@@ -0,0 +1,220 @@
+/**
+ * @fileoverview Backfill Price Data from Firestore Shards to BigQuery
+ * 
+ * This function reads existing price data from Firestore shards
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ * 
+ * Features:
+ *   - Reads from Firestore shards: asset_prices/shard_0, shard_1, etc.
+ *   - Transforms Firestore structure to BigQuery rows
+ *   - Handles compressed Firestore data
+ *   - Uses load jobs (free) for efficient batching
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensureAssetPricesTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Convert Firestore Timestamp to ISO string
+ */
+function convertTimestamp(ts) {
+    if (!ts) return new Date().toISOString();
+    if (ts.toDate) {
+        return ts.toDate().toISOString();
+    }
+    if (ts instanceof Date) {
+        return ts.toISOString();
+    }
+    if (typeof ts === 'string') {
+        return ts;
+    }
+    return new Date().toISOString();
+}
+
+/**
+ * Backfill price data from Firestore shards to BigQuery
+ */
+async function backfillPriceData(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting price data backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensureAssetPricesTable(logger);
+        
+        const priceCollection = db.collection('asset_prices');
+        
+        // Get all shard documents
+        logger.log('INFO', '[Backfill] Fetching price shard documents from Firestore...');
+        const snapshot = await priceCollection.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No price shard documents found in Firestore');
+            return { success: false, message: 'No shards found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} price shard documents`);
+        
+        let totalRows = 0;
+        let processedShards = 0;
+        let skippedInstruments = 0;
+        
+        // Process each shard document
+        for (const doc of snapshot.docs) {
+            const shardId = doc.id; // e.g., "shard_0", "shard_1"
+            
+            try {
+                const shardData = tryDecompress(doc.data());
+                
+                // Process each instrument in the shard
+                for (const [instrumentId, instrumentData] of Object.entries(shardData)) {
+                    // Skip metadata fields
+                    if (instrumentId.startsWith('_')) continue;
+                    
+                    if (!instrumentData || !instrumentData.prices) {
+                        skippedInstruments++;
+                        continue;
+                    }
+                    
+                    const ticker = instrumentData.ticker || `unknown_${instrumentId}`;
+                    const prices = instrumentData.prices || {};
+                    const lastUpdated = convertTimestamp(instrumentData.lastUpdated);
+                    
+                    // Transform to BigQuery rows (one row per date)
+                    const bigqueryRows = [];
+                    
+                    for (const [dateStr, price] of Object.entries(prices)) {
+                        // Filter by date range if provided
+                        if (startDate && endDate) {
+                            if (dateStr < startDate || dateStr > endDate) {
+                                continue;
+                            }
+                        }
+                        
+                        // Validate date format (YYYY-MM-DD)
+                        if (!/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) {
+                            logger.log('WARN', `[Backfill] Skipping invalid date format: ${dateStr} for instrument ${instrumentId}`);
+                            continue;
+                        }
+                        
+                        bigqueryRows.push({
+                            date: dateStr,
+                            instrument_id: parseInt(instrumentId, 10),
+                            ticker: ticker,
+                            price: typeof price === 'number' ? price : null,
+                            open: null, // Firestore shards only store closing price
+                            high: null,
+                            low: null,
+                            close: typeof price === 'number' ? price : null,
+                            volume: null,
+                            fetched_at: lastUpdated
+                        });
+                    }
+                    
+                    if (bigqueryRows.length === 0) {
+                        skippedInstruments++;
+                        continue;
+                    }
+                    
+                    // Write to BigQuery using load jobs (free, batched)
+                    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                    await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                    
+                    totalRows += bigqueryRows.length;
+                }
+                
+                processedShards++;
+                
+                if (processedShards % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedShards} shards, ${totalRows} rows so far...`);
+                }
+            } catch (shardError) {
+                logger.log('ERROR', `[Backfill] Failed to process shard ${shardId}: ${shardError.message}`);
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Price data backfill complete: ${processedShards} shards processed, ${totalRows} rows, ${skippedInstruments} instruments skipped`);
+        
+        return { success: true, processedShards, totalRows, skippedInstruments };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Price data backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillPriceDataMain(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting Price Data backfill...');
+    
+    try {
+        const result = await backfillPriceData(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI handling
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillPriceDataMain(startDate, endDate)
+        .then(result => {
+            console.log('\n✅ Backfill completed successfully!');
+            console.log('Results:', JSON.stringify(result, null, 2));
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('\n❌ Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillPriceData, backfillPriceDataMain };

package/functions/maintenance/backfill-watchlist-membership/index.js

@@ -0,0 +1,190 @@
+/**
+ * @fileoverview Backfill Watchlist Membership from Firestore to BigQuery
+ * 
+ * This function reads existing watchlist membership data from Firestore (WatchlistMembershipData/{date})
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ *   node index.js (backfills all dates)
+ * 
+ * Features:
+ *   - Backfills watchlist membership for date range (or all dates)
+ *   - Uses load jobs (free) for efficient batching
+ *   - Handles compressed Firestore data
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensureWatchlistMembershipTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill watchlist membership from Firestore to BigQuery for a date range
+ */
+async function backfillWatchlistMembership(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting watchlist membership backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensureWatchlistMembershipTable(logger);
+        
+        const membershipCollection = db.collection('WatchlistMembershipData');
+        
+        // Get all membership documents
+        logger.log('INFO', '[Backfill] Fetching watchlist membership documents from Firestore...');
+        const snapshot = await membershipCollection.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No watchlist membership documents found in Firestore');
+            return { success: false, message: 'No watchlist membership found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} watchlist membership documents`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each document
+        for (const doc of snapshot.docs) {
+            const dateStr = doc.id; // Document ID is the date (YYYY-MM-DD)
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                const data = tryDecompress(doc.data());
+                const { date, lastUpdated, ...watchlistMembership } = data;
+                
+                if (!watchlistMembership || Object.keys(watchlistMembership).length === 0) {
+                    logger.log('WARN', `[Backfill] No watchlist membership in document for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Transform to BigQuery rows
+                const lastUpdatedTimestamp = lastUpdated 
+                    ? (lastUpdated.toDate ? lastUpdated.toDate().toISOString() : lastUpdated)
+                    : new Date().toISOString();
+                
+                const bigqueryRows = [];
+                for (const [piId, membershipData] of Object.entries(watchlistMembership)) {
+                    if (membershipData && typeof membershipData === 'object') {
+                        bigqueryRows.push({
+                            date: dateStr,
+                            pi_id: parseInt(piId, 10),
+                            total_users: membershipData.totalUsers || null,
+                            public_watchlist_count: membershipData.publicWatchlistCount || null,
+                            private_watchlist_count: membershipData.privateWatchlistCount || null,
+                            users: membershipData.users || [],
+                            last_updated: lastUpdatedTimestamp
+                        });
+                    }
+                }
+                
+                if (bigqueryRows.length === 0) {
+                    logger.log('WARN', `[Backfill] No valid watchlist membership rows for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'watchlist_membership', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process watchlist membership for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Watchlist membership backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Watchlist membership backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillWatchlistMembershipData(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting Watchlist Membership backfill...');
+    
+    try {
+        const result = await backfillWatchlistMembership(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI execution
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillWatchlistMembershipData(startDate, endDate)
+        .then(result => {
+            console.log('Backfill result:', result);
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillWatchlistMembershipData };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.719",
+  "version": "1.0.720",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -25,7 +25,12 @@
     "functions/maintenance/backfill-instrument-insights",
     "functions/maintenance/backfill-pi-master-list-rankings",
     "functions/maintenance/backfill-task-engine-data",
-    "functions/maintenance/backfill-ticker-mappings"
+    "functions/maintenance/backfill-ticker-mappings",
+    "functions/maintenance/backfill-price-data-from-firestore",
+    "functions/maintenance/backfill-pi-alert-history",
+    "functions/maintenance/backfill-pi-page-views",
+    "functions/maintenance/backfill-pi-ratings",
+    "functions/maintenance/backfill-watchlist-membership"
   ],
   "keywords": [
     "bulltrackers",