bulltrackers-module 1.0.717 → 1.0.718

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -513,6 +513,63 @@ const manageUserWatchlist = async (db, userId, instruction, payload = {}) => {
         // 6. Commit Batch
         await batch.commit();
         
+        // 7. Write to BigQuery (after Firestore commit to get final values)
+        if (process.env.BIGQUERY_ENABLED !== 'false' && (addedItems.length > 0 || removedItems.length > 0)) {
+            try {
+                const { ensureWatchlistMembershipTable, insertRowsWithMerge } = require('../../../core/utils/bigquery_utils');
+                await ensureWatchlistMembershipTable();
+                
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                const bigqueryRows = [];
+                
+                // Process all affected PIs (both added and removed)
+                const affectedPIs = new Set([
+                    ...addedItems.map(item => String(item.cid)),
+                    ...removedItems.map(item => String(item.cid))
+                ]);
+                
+                // Read updated documents from subcollection for each affected PI
+                for (const piId of affectedPIs) {
+                    const masterRef = db.collection('WatchlistMembershipData')
+                        .doc(todayStr).collection('popular_investors').doc(piId);
+                    
+                    const piDoc = await masterRef.get();
+                    if (piDoc.exists) {
+                        const piData = piDoc.data();
+                        
+                        // Transform to BigQuery format
+                        const lastUpdated = piData.lastUpdated 
+                            ? (piData.lastUpdated.toDate ? piData.lastUpdated.toDate().toISOString() : piData.lastUpdated)
+                            : new Date().toISOString();
+                        
+                        bigqueryRows.push({
+                            date: todayStr,
+                            pi_id: parseInt(piId, 10),
+                            total_users: piData.totalUsers || 0,
+                            public_watchlist_count: piData.publicWatchlistCount || 0,
+                            private_watchlist_count: piData.privateWatchlistCount || 0,
+                            users: Array.isArray(piData.users) ? piData.users : [],
+                            last_updated: lastUpdated
+                        });
+                    }
+                }
+                
+                if (bigqueryRows.length > 0) {
+                    // Use MERGE to update existing rows or insert new
+                    await insertRowsWithMerge(
+                        datasetId,
+                        'watchlist_membership',
+                        bigqueryRows,
+                        ['date', 'pi_id'], // Key fields for MERGE
+                        console // Simple logger
+                    );
+                }
+            } catch (bqError) {
+                console.error(`[manageUserWatchlist] BigQuery write failed: ${bqError.message}`);
+                // Don't throw - Firestore write succeeded, BigQuery is secondary
+            }
+        }
+        
         return { 
             success: true, 
             id: watchlistId, 
@@ -1583,6 +1640,51 @@ const trackPopularInvestorView = async (db, piId, viewerId = null, viewerType =
     batch.set(globalDayRef, globalUpdate, { merge: true });
 
     await batch.commit();
+    
+    // 5. Write to BigQuery (after Firestore commit to get final values)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { ensurePIPageViewsTable, insertRowsWithMerge } = require('../../../core/utils/bigquery_utils');
+            await ensurePIPageViewsTable();
+            
+            // Read the updated document to get final values (after increments)
+            const updatedDoc = await globalDayRef.get();
+            if (updatedDoc.exists) {
+                const data = updatedDoc.data();
+                const piData = data[piId];
+                
+                if (piData) {
+                    // Transform to BigQuery format
+                    const lastUpdated = piData.lastUpdated 
+                        ? (piData.lastUpdated.toDate ? piData.lastUpdated.toDate().toISOString() : piData.lastUpdated)
+                        : new Date().toISOString();
+                    
+                    const bigqueryRow = {
+                        date: todayStr,
+                        pi_id: parseInt(piId, 10),
+                        total_views: piData.totalViews || 0,
+                        unique_viewers: piData.uniqueViewers || 0,
+                        views_by_user: piData.viewsByUser || {},
+                        last_updated: lastUpdated
+                    };
+                    
+                    // Use MERGE to update existing row or insert new
+                    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                    await insertRowsWithMerge(
+                        datasetId,
+                        'pi_page_views',
+                        [bigqueryRow],
+                        ['date', 'pi_id'], // Key fields for MERGE
+                        console // Simple logger
+                    );
+                }
+            }
+        } catch (bqError) {
+            console.error(`[trackPopularInvestorView] BigQuery write failed: ${bqError.message}`);
+            // Don't throw - Firestore write succeeded, BigQuery is secondary
+        }
+    }
+    
     return { success: true };
 };
 // ==========================================
@@ -3494,6 +3596,152 @@ const subscribeToAllWatchlistPIs = async (db, userId, watchlistId, alertTypes =
     }
 };
 
+/**
+ * Helper function for BigQuery caching pattern:
+ * 1. Check Firestore cache (with TTL)
+ * 2. If missing/expired, fetch from BigQuery
+ * 3. Cache result in Firestore with 10-minute TTL
+ * @param {Object} firestore - Firestore instance
+ * @param {string} cacheCollection - Firestore collection for cache (e.g., 'PIRatingsData')
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @param {Function} bigqueryFetcher - Async function that fetches from BigQuery
+ * @param {string} dataType - Data type name for logging
+ * @returns {Promise<Object|null>} Cached or fetched data
+ */
+async function fetchWithBigQueryCache(firestore, cacheCollection, dateStr, bigqueryFetcher, dataType = 'data') {
+    const CACHE_TTL_MINUTES = 10;
+    const CACHE_TTL_MS = CACHE_TTL_MINUTES * 60 * 1000;
+    
+    try {
+        // 1. Check Firestore cache
+        const cacheDocRef = firestore.collection(cacheCollection).doc(dateStr);
+        const cacheDoc = await cacheDocRef.get();
+        
+        if (cacheDoc.exists) {
+            const cacheData = cacheDoc.data();
+            const cachedAt = cacheData.cachedAt;
+            
+            // Check if cache is still valid
+            if (cachedAt) {
+                const cachedTimestamp = cachedAt.toMillis ? cachedAt.toMillis() : (cachedAt._seconds * 1000);
+                const now = Date.now();
+                const age = now - cachedTimestamp;
+                
+                if (age < CACHE_TTL_MS) {
+                    // Cache is valid, return cached data
+                    const { cachedAt, ...data } = cacheData;
+                    return data;
+                }
+            }
+        }
+        
+        // 2. Cache missing or expired, fetch from BigQuery
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const bigqueryData = await bigqueryFetcher();
+                
+                if (bigqueryData) {
+                    // 3. Cache the result in Firestore with TTL
+                    await cacheDocRef.set({
+                        ...bigqueryData,
+                        cachedAt: FieldValue.serverTimestamp()
+                    }, { merge: true });
+                    
+                    return bigqueryData;
+                }
+            } catch (error) {
+                console.error(`[API] BigQuery fetch failed for ${dataType} (${dateStr}): ${error.message}`);
+                // If BigQuery fails, return cached data if available (even if expired)
+                if (cacheDoc.exists) {
+                    const cacheData = cacheDoc.data();
+                    const { cachedAt, ...data } = cacheData;
+                    return data;
+                }
+            }
+        }
+        
+        // 4. Fallback: return cached data even if expired, or null
+        if (cacheDoc.exists) {
+            const cacheData = cacheDoc.data();
+            const { cachedAt, ...data } = cacheData;
+            return data;
+        }
+        
+        return null;
+    } catch (error) {
+        console.error(`[API] Error in fetchWithBigQueryCache for ${dataType} (${dateStr}): ${error.message}`);
+        return null;
+    }
+}
+
+/**
+ * Fetch PI Ratings data with BigQuery caching
+ * @param {Object} firestore - Firestore instance
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @returns {Promise<Object|null>} Ratings data
+ */
+async function fetchPIRatings(firestore, dateStr) {
+    const { queryPIRatings } = require('../../../core/utils/bigquery_utils');
+    return await fetchWithBigQueryCache(
+        firestore,
+        'PIRatingsData',
+        dateStr,
+        () => queryPIRatings(dateStr),
+        'PI Ratings'
+    );
+}
+
+/**
+ * Fetch PI Page Views data with BigQuery caching
+ * @param {Object} firestore - Firestore instance
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @returns {Promise<Object|null>} Page views data
+ */
+async function fetchPIPageViews(firestore, dateStr) {
+    const { queryPIPageViews } = require('../../../core/utils/bigquery_utils');
+    return await fetchWithBigQueryCache(
+        firestore,
+        'PIPageViewsData',
+        dateStr,
+        () => queryPIPageViews(dateStr),
+        'PI Page Views'
+    );
+}
+
+/**
+ * Fetch Watchlist Membership data with BigQuery caching
+ * @param {Object} firestore - Firestore instance
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @returns {Promise<Object|null>} Watchlist membership data
+ */
+async function fetchWatchlistMembership(firestore, dateStr) {
+    const { queryWatchlistMembership } = require('../../../core/utils/bigquery_utils');
+    return await fetchWithBigQueryCache(
+        firestore,
+        'WatchlistMembershipData',
+        dateStr,
+        () => queryWatchlistMembership(dateStr),
+        'Watchlist Membership'
+    );
+}
+
+/**
+ * Fetch PI Alert History data with BigQuery caching
+ * @param {Object} firestore - Firestore instance
+ * @param {string} dateStr - Date string (YYYY-MM-DD)
+ * @returns {Promise<Object|null>} Alert history data
+ */
+async function fetchPIAlertHistory(firestore, dateStr) {
+    const { queryPIAlertHistory } = require('../../../core/utils/bigquery_utils');
+    return await fetchWithBigQueryCache(
+        firestore,
+        'PIAlertHistoryData',
+        dateStr,
+        () => queryPIAlertHistory(dateStr),
+        'PI Alert History'
+    );
+}
+
 module.exports = {
     latestUserCentricSnapshot,
     pageCollection,
@@ -3542,5 +3790,9 @@ module.exports = {
     unsubscribeFromAlerts,
     getWatchlistTriggerCounts,
     subscribeToAllWatchlistPIs,
-    queryDynamicWatchlistMatches
+    queryDynamicWatchlistMatches,
+    fetchPIRatings,
+    fetchPIPageViews,
+    fetchWatchlistMembership,
+    fetchPIAlertHistory
 };

package/functions/computation-system/utils/data_loader.js

@@ -376,8 +376,58 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'social', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
-    logger.log('INFO', `Loading and partitioning social data for ${dateString}`);
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { querySocialData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await querySocialData(dateString, null, null, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for social data (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform BigQuery data to expected format: { generic: {}, pi: {}, signedIn: {} }
+                // BigQuery returns: { userId: { posts_data: { posts: {...}, postCount: N }, user_type: '...' } }
+                const result = { generic: {}, pi: {}, signedIn: {} };
+                
+                for (const [userId, userData] of Object.entries(bigqueryData)) {
+                    const userType = userData.user_type || 'UNKNOWN';
+                    
+                    // Handle posts_data - may be object (parsed JSON) or string (needs parsing)
+                    let postsData = userData.posts_data || {};
+                    if (typeof postsData === 'string') {
+                        try {
+                            postsData = JSON.parse(postsData);
+                        } catch (e) {
+                            logger.log('WARN', `[DataLoader] Failed to parse posts_data for user ${userId}: ${e.message}`);
+                            continue;
+                        }
+                    }
+                    
+                    // Extract posts map from posts_data structure: { posts: {...}, postCount: N }
+                    const posts = postsData.posts || {};
+                    
+                    // Partition by user type
+                    if (userType === 'POPULAR_INVESTOR') {
+                        result.pi[userId] = posts;
+                    } else if (userType === 'SIGNED_IN_USER') {
+                        result.signedIn[userId] = posts;
+                    } else {
+                        // Generic/unknown user types go to generic
+                        result.generic[userId] = posts;
+                    }
+                }
+                
+                logger.log('INFO', `[DataLoader] ✅ Loaded Social Data from BigQuery: ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} Signed-In`);
+                return result;
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery social query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
+    logger.log('INFO', `Loading and partitioning social data for ${dateString} (Firestore)`);
     
     const result = { generic: {}, pi: {}, signedIn: {} };
 
@@ -874,31 +924,33 @@ async function loadPIRatings(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'ratings', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
-    logger.log('INFO', `Loading PI Ratings (Raw Logs) for ${dateString}`);
-    try {
-        const shardsColRef = db.collection('PiReviews').doc(dateString).collection('shards');
-        const shardDocs = await withRetry(() => shardsColRef.listDocuments(), `listRatingShards(${dateString})`);
-
-        if (!shardDocs || shardDocs.length === 0) {
-            logger.log('WARN', `No rating shards found for ${dateString}`);
-            return {};
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIRatings } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIRatings(dateString, logger);
+            if (bigqueryData) {
+                logger.log('INFO', `[DataLoader] ✅ Loaded PI Ratings from BigQuery for ${dateString}`);
+                return bigqueryData;
+            }
+        } catch (error) {
+            logger.log('WARN', `[DataLoader] BigQuery PI Ratings query failed, falling back to Firestore: ${error.message}`);
         }
+    }
 
-        const rawReviewsByPi = {}; 
-        for (const docRef of shardDocs) {
-            const docSnap = await docRef.get();
-            if (!docSnap.exists) continue;
-            const rawData = tryDecompress(docSnap.data());
-            Object.values(rawData).forEach(entry => {
-                if (entry && typeof entry === 'object' && entry.piCid && entry.rating !== undefined) {
-                    if (!rawReviewsByPi[entry.piCid]) rawReviewsByPi[entry.piCid] = [];
-                    rawReviewsByPi[entry.piCid].push(entry);
-                }
-            });
+    // 3. FIRESTORE FALLBACK
+    const collectionName = config.piRatingsCollection || 'PIRatingsData';
+    logger.log('INFO', `Loading PI Ratings from Firestore for ${dateString}`);
+    try {
+        const docRef = db.collection(collectionName).doc(dateString);
+        const docSnap = await withRetry(() => docRef.get(), `getPIRatings(${dateString})`);
+        if (!docSnap.exists) { 
+            logger.log('WARN', `PI Ratings not found for ${dateString}`); 
+            return {}; 
         }
-        logger.log('INFO', `Loaded raw reviews for ${Object.keys(rawReviewsByPi).length} PIs.`);
-        return rawReviewsByPi; 
+        const data = tryDecompress(docSnap.data());
+        const { date, lastUpdated, ...piRatings } = data;
+        return piRatings; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Ratings: ${error.message}`);
         return {};
@@ -914,9 +966,23 @@ async function loadPIPageViews(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'page_views', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIPageViews } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIPageViews(dateString, logger);
+            if (bigqueryData) {
+                logger.log('INFO', `[DataLoader] ✅ Loaded PI Page Views from BigQuery for ${dateString}`);
+                return bigqueryData;
+            }
+        } catch (error) {
+            logger.log('WARN', `[DataLoader] BigQuery PI Page Views query failed, falling back to Firestore: ${error.message}`);
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.piPageViewsCollection || 'PIPageViewsData';
-    logger.log('INFO', `Loading PI Page Views for ${dateString}`);
+    logger.log('INFO', `Loading PI Page Views from Firestore for ${dateString}`);
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getPIPageViews(${dateString})`);
@@ -939,9 +1005,23 @@ async function loadWatchlistMembership(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'watchlist', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryWatchlistMembership } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryWatchlistMembership(dateString, logger);
+            if (bigqueryData) {
+                logger.log('INFO', `[DataLoader] ✅ Loaded Watchlist Membership from BigQuery for ${dateString}`);
+                return bigqueryData;
+            }
+        } catch (error) {
+            logger.log('WARN', `[DataLoader] BigQuery Watchlist Membership query failed, falling back to Firestore: ${error.message}`);
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.watchlistMembershipCollection || 'WatchlistMembershipData';
-    logger.log('INFO', `Loading Watchlist Membership for ${dateString}`);
+    logger.log('INFO', `Loading Watchlist Membership from Firestore for ${dateString}`);
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getWatchlistMembership(${dateString})`);
@@ -964,9 +1044,23 @@ async function loadPIAlertHistory(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'alerts', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIAlertHistory } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIAlertHistory(dateString, logger);
+            if (bigqueryData) {
+                logger.log('INFO', `[DataLoader] ✅ Loaded PI Alert History from BigQuery for ${dateString}`);
+                return bigqueryData;
+            }
+        } catch (error) {
+            logger.log('WARN', `[DataLoader] BigQuery PI Alert History query failed, falling back to Firestore: ${error.message}`);
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.piAlertHistoryCollection || 'PIAlertHistoryData';
-    logger.log('INFO', `Loading PI Alert History for ${dateString}`);
+    logger.log('INFO', `Loading PI Alert History from Firestore for ${dateString}`);
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getPIAlertHistory(${dateString})`);

package/functions/core/utils/bigquery_utils.js

@@ -630,6 +630,42 @@ const SCHEMAS = {
         { name: 'instrument_id', type: 'INT64', mode: 'REQUIRED' },
         { name: 'ticker', type: 'STRING', mode: 'REQUIRED' },
         { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    pi_ratings: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'pi_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'average_rating', type: 'FLOAT64', mode: 'NULLABLE' },
+        { name: 'total_ratings', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'ratings_by_user', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    pi_page_views: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'pi_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'total_views', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'unique_viewers', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'views_by_user', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    watchlist_membership: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'pi_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'total_users', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'public_watchlist_count', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'private_watchlist_count', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'users', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
+    ],
+    pi_alert_history: [
+        { name: 'date', type: 'DATE', mode: 'REQUIRED' },
+        { name: 'pi_id', type: 'INT64', mode: 'REQUIRED' },
+        { name: 'alert_type', type: 'STRING', mode: 'REQUIRED' },
+        { name: 'triggered', type: 'BOOLEAN', mode: 'NULLABLE' },
+        { name: 'trigger_count', type: 'INT64', mode: 'NULLABLE' },
+        { name: 'triggered_for', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'metadata', type: 'JSON', mode: 'NULLABLE' },
+        { name: 'last_triggered', type: 'TIMESTAMP', mode: 'NULLABLE' },
+        { name: 'last_updated', type: 'TIMESTAMP', mode: 'REQUIRED' }
     ]
 };
 
@@ -838,6 +874,94 @@ async function ensureTickerMappingsTable(logger = null) {
     );
 }
 
+/**
+ * Ensure pi_ratings table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensurePIRatingsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'pi_ratings';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['pi_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure pi_page_views table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensurePIPageViewsTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'pi_page_views';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['pi_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure watchlist_membership table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensureWatchlistMembershipTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'watchlist_membership';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['pi_id']
+        },
+        logger
+    );
+}
+
+/**
+ * Ensure pi_alert_history table exists
+ * @param {object} logger - Logger instance
+ * @returns {Promise<Table>}
+ */
+async function ensurePIAlertHistoryTable(logger = null) {
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tableId = 'pi_alert_history';
+    const schema = getSchema(tableId);
+    
+    return await ensureTableExists(
+        datasetId,
+        tableId,
+        schema,
+        {
+            partitionField: 'date',
+            clusterFields: ['pi_id', 'alert_type']
+        },
+        logger
+    );
+}
+
 /**
  * Query portfolio data from BigQuery
  * @param {string} dateStr - Date string (YYYY-MM-DD)
@@ -1082,16 +1206,18 @@ async function querySocialData(dateStr, userIds = null, userTypes = null, logger
     const tablePath = `${datasetId}.social_post_snapshots`;
     
     try {
-        const conditions = [`date = '${dateStr}'`];
+        // Build WHERE clause with parameterized queries (SQL injection safe)
+        const conditions = [`date = @dateStr`];
+        const params = { dateStr: dateStr };
         
         if (userIds && userIds.length > 0) {
-            const userIdList = userIds.map(id => String(id)).join(',');
-            conditions.push(`user_id IN (${userIdList})`);
+            conditions.push(`user_id IN UNNEST(@userIds)`);
+            params.userIds = userIds.map(id => parseInt(id, 10));
         }
         
         if (userTypes && userTypes.length > 0) {
-            const typeList = userTypes.map(t => `'${t.toUpperCase()}'`).join(',');
-            conditions.push(`user_type IN (${typeList})`);
+            conditions.push(`user_type IN UNNEST(@userTypes)`);
+            params.userTypes = userTypes.map(t => t.toUpperCase());
         }
         
         const whereClause = conditions.join(' AND ');
@@ -1110,7 +1236,7 @@ async function querySocialData(dateStr, userIds = null, userTypes = null, logger
             logger.log('INFO', `[BigQuery] 🔍 Querying social posts from ${tablePath} for date ${dateStr}${userTypes ? ` (types: ${userTypes.join(',')})` : ''}${userIds ? ` (${userIds.length} users)` : ''}`);
         }
         
-        const rows = await query(sqlQuery, {}, logger);
+        const rows = await query(sqlQuery, { params }, logger);
         
         if (!rows || rows.length === 0) {
             if (logger) logger.log('INFO', `[BigQuery] No social data found in ${tablePath} for ${dateStr}`);
@@ -1711,6 +1837,285 @@ async function queryTickerMappings(logger = null) {
     }
 }
 
+/**
+ * Query PI ratings from BigQuery for a specific date
+ * Returns data in format: { piId: { averageRating, totalRatings, ratingsByUser } }
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Ratings data map, or null if not found/error
+ */
+async function queryPIRatings(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] PI ratings query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.pi_ratings`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                pi_id,
+                average_rating,
+                total_ratings,
+                ratings_by_user
+            FROM \`${tablePath}\`
+            WHERE date = @dateStr
+            ORDER BY pi_id ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying PI ratings from ${tablePath} for ${dateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                dateStr: dateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No PI ratings found for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to expected format: { piId: { averageRating, totalRatings, ratingsByUser } }
+        const ratings = {};
+        for (const row of rows) {
+            const piId = String(row.pi_id);
+            ratings[piId] = {
+                averageRating: row.average_rating || 0,
+                totalRatings: row.total_ratings || 0,
+                ratingsByUser: row.ratings_by_user || {}
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved ratings for ${Object.keys(ratings).length} PIs for ${dateStr}`);
+        }
+        
+        return ratings;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] PI ratings query failed for ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query PI page views from BigQuery for a specific date
+ * Returns data in format: { piId: { totalViews, uniqueViewers, viewsByUser } }
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Page views data map, or null if not found/error
+ */
+async function queryPIPageViews(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] PI page views query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.pi_page_views`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                pi_id,
+                total_views,
+                unique_viewers,
+                views_by_user
+            FROM \`${tablePath}\`
+            WHERE date = @dateStr
+            ORDER BY pi_id ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying PI page views from ${tablePath} for ${dateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                dateStr: dateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No PI page views found for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to expected format: { piId: { totalViews, uniqueViewers, viewsByUser } }
+        const pageViews = {};
+        for (const row of rows) {
+            const piId = String(row.pi_id);
+            pageViews[piId] = {
+                totalViews: row.total_views || 0,
+                uniqueViewers: row.unique_viewers || 0,
+                viewsByUser: row.views_by_user || {}
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved page views for ${Object.keys(pageViews).length} PIs for ${dateStr}`);
+        }
+        
+        return pageViews;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] PI page views query failed for ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query watchlist membership from BigQuery for a specific date
+ * Returns data in format: { piId: { totalUsers, users, publicWatchlistCount, privateWatchlistCount } }
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Watchlist membership data map, or null if not found/error
+ */
+async function queryWatchlistMembership(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Watchlist membership query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.watchlist_membership`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                pi_id,
+                total_users,
+                public_watchlist_count,
+                private_watchlist_count,
+                users
+            FROM \`${tablePath}\`
+            WHERE date = @dateStr
+            ORDER BY pi_id ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying watchlist membership from ${tablePath} for ${dateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                dateStr: dateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No watchlist membership found for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to expected format: { piId: { totalUsers, users, publicWatchlistCount, privateWatchlistCount } }
+        const membership = {};
+        for (const row of rows) {
+            const piId = String(row.pi_id);
+            membership[piId] = {
+                totalUsers: row.total_users || 0,
+                users: row.users || [],
+                publicWatchlistCount: row.public_watchlist_count || 0,
+                privateWatchlistCount: row.private_watchlist_count || 0
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved watchlist membership for ${Object.keys(membership).length} PIs for ${dateStr}`);
+        }
+        
+        return membership;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Watchlist membership query failed for ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query PI alert history from BigQuery for a specific date
+ * Returns data in format: { piId: { alertType: { triggered, count, triggeredFor, metadata } } }
+ * @param {string} dateStr - Date (YYYY-MM-DD)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Alert history data map, or null if not found/error
+ */
+async function queryPIAlertHistory(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] PI alert history query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.pi_alert_history`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                pi_id,
+                alert_type,
+                triggered,
+                trigger_count,
+                triggered_for,
+                metadata
+            FROM \`${tablePath}\`
+            WHERE date = @dateStr
+            ORDER BY pi_id ASC, alert_type ASC
+        `;
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying PI alert history from ${tablePath} for ${dateStr}`);
+        }
+        
+        const rows = await query(sqlQuery, {
+            params: {
+                dateStr: dateStr
+            }
+        }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No PI alert history found for ${dateStr}`);
+            return null;
+        }
+        
+        // Transform to expected format: { piId: { alertType: { triggered, count, triggeredFor, metadata } } }
+        const alertHistory = {};
+        for (const row of rows) {
+            const piId = String(row.pi_id);
+            const alertType = row.alert_type;
+            
+            if (!alertHistory[piId]) {
+                alertHistory[piId] = {};
+            }
+            
+            alertHistory[piId][alertType] = {
+                triggered: row.triggered || false,
+                count: row.trigger_count || 0,
+                triggeredFor: row.triggered_for || [],
+                metadata: row.metadata || {}
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved alert history for ${Object.keys(alertHistory).length} PIs for ${dateStr}`);
+        }
+        
+        return alertHistory;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] PI alert history query failed for ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
 module.exports = {
     getBigQueryClient,
     getOrCreateDataset,
@@ -1728,6 +2133,10 @@ module.exports = {
     ensurePIRankingsTable,
     ensureInstrumentInsightsTable,
     ensureTickerMappingsTable,
+    ensurePIRatingsTable,
+    ensurePIPageViewsTable,
+    ensureWatchlistMembershipTable,
+    ensurePIAlertHistoryTable,
     queryPortfolioData,
     queryHistoryData,
     querySocialData,
@@ -1736,6 +2145,10 @@ module.exports = {
     queryPIRankings,
     queryInstrumentInsights,
     queryTickerMappings,
+    queryPIRatings,
+    queryPIPageViews,
+    queryWatchlistMembership,
+    queryPIAlertHistory,
     queryComputationResult,
     queryComputationResultsRange,
     checkExistingRows,

package/functions/maintenance/backfill-ticker-mappings/index.js

@@ -0,0 +1,116 @@
+/**
+ * @fileoverview Backfill Ticker Mappings from Firestore to BigQuery
+ * 
+ * This function reads the single ticker mappings document from Firestore
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js
+ * 
+ * Features:
+ *   - Reads single document from Firestore: instrument_mappings/etoro_to_ticker
+ *   - Transforms to BigQuery rows (one row per instrument)
+ *   - Uses insertRowsWithMerge (MERGE operation) for idempotent writes
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const { 
+    ensureTickerMappingsTable,
+    insertRowsWithMerge
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+/**
+ * Backfill ticker mappings from Firestore to BigQuery
+ */
+async function backfillTickerMappings(logger = console) {
+    logger.log('INFO', '[Backfill] Starting ticker mappings backfill...');
+    
+    try {
+        await ensureTickerMappingsTable(logger);
+        
+        // Read the single document from Firestore
+        logger.log('INFO', '[Backfill] Fetching ticker mappings from Firestore...');
+        const docRef = db.collection('instrument_mappings').doc('etoro_to_ticker');
+        const docSnap = await docRef.get();
+        
+        if (!docSnap.exists) {
+            logger.log('WARN', '[Backfill] Ticker mappings document not found in Firestore');
+            return { success: false, message: 'Document not found' };
+        }
+        
+        const mappingsData = docSnap.data();
+        const instrumentIds = Object.keys(mappingsData);
+        
+        if (instrumentIds.length === 0) {
+            logger.log('WARN', '[Backfill] Ticker mappings document is empty');
+            return { success: false, message: 'Document is empty' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${instrumentIds.length} ticker mappings`);
+        
+        // Transform to BigQuery rows
+        const fetchedAt = new Date().toISOString();
+        const bigqueryRows = instrumentIds.map(instrumentId => {
+            return {
+                instrument_id: parseInt(instrumentId, 10),
+                ticker: String(mappingsData[instrumentId]),
+                last_updated: fetchedAt
+            };
+        });
+        
+        // Write to BigQuery using MERGE (idempotent, can re-run safely)
+        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        const keyFields = ['instrument_id'];
+        await insertRowsWithMerge(datasetId, 'ticker_mappings', bigqueryRows, keyFields, logger);
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Ticker mappings backfill complete: ${bigqueryRows.length} mappings`);
+        
+        return { success: true, totalRows: bigqueryRows.length };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Ticker mappings backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillTickerMappingsMain() {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting Ticker Mappings backfill...');
+    
+    try {
+        const result = await backfillTickerMappings(logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI handling
+if (require.main === module) {
+    backfillTickerMappingsMain()
+        .then(result => {
+            console.log('\n✅ Backfill completed successfully!');
+            console.log('Results:', JSON.stringify(result, null, 2));
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('\n❌ Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillTickerMappings, backfillTickerMappingsMain };

package/index.js

@@ -67,6 +67,10 @@ const { backfillPIMasterListRankings }                    = require('./functions
 const { backfillInstrumentInsights }                      = require('./functions/maintenance/backfill-instrument-insights/index');
 const { backfillTickerMappings }                          = require('./functions/maintenance/backfill-ticker-mappings/index');
 const { backfillPriceData }                               = require('./functions/maintenance/backfill-price-data-from-firestore/index');
+const { backfillPIRatings }                               = require('./functions/maintenance/backfill-pi-ratings/index');
+const { backfillPIPageViews }                             = require('./functions/maintenance/backfill-pi-page-views/index');
+const { backfillWatchlistMembershipData }                 = require('./functions/maintenance/backfill-watchlist-membership/index');
+const { backfillPIAlertHistory }                          = require('./functions/maintenance/backfill-pi-alert-history/index');
 
 // Alert System
 const { handleAlertTrigger, handleComputationResultWrite, checkAndSendAllClearNotifications } = require('./functions/alert-system/index');
@@ -139,7 +143,11 @@ const maintenance = {
   backfillPIMasterListRankings,
   backfillInstrumentInsights,
   backfillTickerMappings,
-  backfillPriceData
+  backfillPriceData,
+  backfillPIRatings,
+  backfillPIPageViews,
+  backfillWatchlistMembershipData,
+  backfillPIAlertHistory
 };
 
 const proxy = { handlePost };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.717",
+  "version": "1.0.718",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -24,7 +24,8 @@
     "functions/alert-system/",
     "functions/maintenance/backfill-instrument-insights",
     "functions/maintenance/backfill-pi-master-list-rankings",
-    "functions/maintenance/backfill-task-engine-data"
+    "functions/maintenance/backfill-task-engine-data",
+    "functions/maintenance/backfill-ticker-mappings"
   ],
   "keywords": [
     "bulltrackers",