bulltrackers-module 1.0.710 → 1.0.713

package/functions/price-backfill/helpers/handler_helpers.js

@@ -1,14 +1,17 @@
 /**
  * @fileoverview Main pipe: pipe.maintenance.runBackfillAssetPrices
  * A one-time function to backfill historical price data from eToro's
- * candle API into the new sharded `asset_prices` collection.
+ * candle API into BigQuery `asset_prices` table.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit'); 
 const CONCURRENT_REQUESTS = 10;
 const DAYS_TO_FETCH = 365;
-const SHARD_SIZE = 40;
+const SHARD_SIZE = 40; // Legacy, kept for Firestore compatibility if needed
+
+// BigQuery batch size for load jobs (100MB limit, but we batch smaller for efficiency)
+const BIGQUERY_BATCH_SIZE = 10000; // rows per batch
 
 /**
  * Main pipe: pipe.maintenance.runBackfillAssetPrices
@@ -42,14 +45,60 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
                 headerManager.updatePerformance(selectedHeader.id, wasSuccess);
                 const data = await response.json();
                 const candles = data?.Candles?.[0]?.Candles;
-                if (!Array.isArray(candles) || candles.length === 0) { logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`); return; }
-                const prices = {};
-                for (const candle of candles) { const dateKey = candle.FromDate.substring(0, 10); prices[dateKey] = candle.Close; }
-                const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
-                const docRef = db.collection('asset_prices').doc(shardId); 
-                const payload = { [instrumentId]: { ticker: ticker, prices: prices, lastUpdated: FieldValue.serverTimestamp() } };
-                await docRef.set(payload, { merge: true });
-                logger.log('TRACE', `[PriceBackfill] Successfully stored data for ${ticker} (${instrumentId}) in ${shardId}`);
+                if (!Array.isArray(candles) || candles.length === 0) { 
+                    logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`); 
+                    return; 
+                }
+                
+                // Transform candles to BigQuery rows
+                const fetchedAt = new Date().toISOString();
+                const bigqueryRows = candles.map(candle => {
+                    const dateStr = candle.FromDate.substring(0, 10); // Extract YYYY-MM-DD
+                    return {
+                        date: dateStr,
+                        instrument_id: parseInt(instrumentId, 10),
+                        ticker: ticker,
+                        price: candle.Close || null,
+                        open: candle.Open || null,
+                        high: candle.High || null,
+                        low: candle.Low || null,
+                        close: candle.Close || null,
+                        volume: candle.Volume || null,
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                // Write to BigQuery using load jobs (free, batched)
+                if (process.env.BIGQUERY_ENABLED !== 'false') {
+                    try {
+                        const { insertRows, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
+                        await ensureAssetPricesTable(logger);
+                        
+                        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                        await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                        
+                        logger.log('TRACE', `[PriceBackfill] Successfully stored ${bigqueryRows.length} price records for ${ticker} (${instrumentId}) to BigQuery`);
+                    } catch (bqError) {
+                        logger.log('ERROR', `[PriceBackfill] BigQuery write failed for ${ticker} (${instrumentId}): ${bqError.message}`);
+                        // Continue - don't fail the entire backfill for one instrument
+                    }
+                }
+                
+                // Also write to Firestore for backward compatibility (if needed)
+                // This can be removed once all readers are migrated to BigQuery
+                if (process.env.FIRESTORE_PRICE_BACKFILL !== 'false') {
+                    const prices = {};
+                    for (const candle of candles) { 
+                        const dateKey = candle.FromDate.substring(0, 10); 
+                        prices[dateKey] = candle.Close; 
+                    }
+                    const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
+                    const docRef = db.collection('asset_prices').doc(shardId); 
+                    const payload = { [instrumentId]: { ticker: ticker, prices: prices, lastUpdated: FieldValue.serverTimestamp() } };
+                    await docRef.set(payload, { merge: true });
+                    logger.log('TRACE', `[PriceBackfill] Also stored data for ${ticker} (${instrumentId}) in Firestore ${shardId}`);
+                }
+                
                 successCount++;
             } catch (err) {
                 logger.log('ERROR', `[PriceBackfill] Failed to process instrument ${instrumentId}`, { err: err.message });

package/functions/root-data-indexer/index.js

@@ -245,6 +245,56 @@ exports.runRootDataIndexer = async (config, dependencies) => {
 
             // --- Define Refs & Check Paths ---
             
+            // =========================================================================
+            // BIGQUERY FIRST: Check BigQuery tables for data availability
+            // =========================================================================
+            let bigqueryHasPortfolio = false;
+            let bigqueryHasHistory = false;
+            let bigqueryHasSocial = false;
+            let bigqueryHasInsights = false;
+            let bigqueryHasPrices = false;
+            let bigqueryHasPIRankings = false;
+            let bigqueryHasPIMasterList = false;
+            
+            if (process.env.BIGQUERY_ENABLED !== 'false') {
+                try {
+                    const { 
+                        queryPortfolioData, 
+                        queryHistoryData, 
+                        querySocialData,
+                        queryInstrumentInsights,
+                        queryAssetPrices,
+                        queryPIRankings,
+                        queryPIMasterList
+                    } = require('../../core/utils/bigquery_utils');
+                    
+                    // Check BigQuery for today's data (or target date)
+                    const [portfolioData, historyData, socialData, insightsData, pricesData, rankingsData, masterListData] = await Promise.all([
+                        queryPortfolioData(dateStr, null, null, logger).catch(() => null),
+                        queryHistoryData(dateStr, null, null, logger).catch(() => null),
+                        querySocialData(dateStr, null, null, logger).catch(() => null),
+                        queryInstrumentInsights(dateStr, logger).catch(() => null),
+                        queryAssetPrices(dateStr, dateStr, null, logger).catch(() => null), // Check for specific date
+                        queryPIRankings(dateStr, logger).catch(() => null),
+                        queryPIMasterList(logger).catch(() => null)
+                    ]);
+                    
+                    bigqueryHasPortfolio = portfolioData && Object.keys(portfolioData).length > 0;
+                    bigqueryHasHistory = historyData && Object.keys(historyData).length > 0;
+                    bigqueryHasSocial = socialData && Object.keys(socialData).length > 0;
+                    bigqueryHasInsights = insightsData && Array.isArray(insightsData) && insightsData.length > 0;
+                    bigqueryHasPrices = pricesData && Object.keys(pricesData).length > 0;
+                    bigqueryHasPIRankings = rankingsData && Array.isArray(rankingsData) && rankingsData.length > 0;
+                    bigqueryHasPIMasterList = masterListData && Object.keys(masterListData).length > 0;
+                    
+                    if (bigqueryHasPortfolio || bigqueryHasHistory || bigqueryHasSocial || bigqueryHasInsights || bigqueryHasPrices || bigqueryHasPIRankings) {
+                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✅ Found data in BigQuery: portfolio=${bigqueryHasPortfolio}, history=${bigqueryHasHistory}, social=${bigqueryHasSocial}, insights=${bigqueryHasInsights}, prices=${bigqueryHasPrices}, rankings=${bigqueryHasPIRankings}`);
+                    }
+                } catch (bqError) {
+                    logger.log('WARN', `[RootDataIndexer/${dateStr}] BigQuery check failed, using Firestore fallback: ${bqError.message}`);
+                }
+            }
+            
             // 1. Standard Retail
             // Path: {normalPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
             const normPortPartsRef = db.collection(safeCollections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
@@ -435,25 +485,29 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             logger.log('INFO', `[RootDataIndexer/${dateStr}] Social check results - PI: ${foundPISocial}, Signed-in: ${foundSignedInSocial}`);
 
             // --- Assign to Availability ---
-            availability.details.normalPortfolio     = normPortExists;
-            availability.details.speculatorPortfolio = specPortExists;
-            availability.details.normalHistory       = normHistExists;
-            availability.details.speculatorHistory   = specHistExists;
-            availability.details.piRankings          = piRankingsSnap.exists;
+            // Use BigQuery results if available, otherwise fall back to Firestore
+            availability.details.normalPortfolio     = bigqueryHasPortfolio || normPortExists;
+            availability.details.speculatorPortfolio = bigqueryHasPortfolio || specPortExists;
+            availability.details.normalHistory       = bigqueryHasHistory || normHistExists;
+            availability.details.speculatorHistory   = bigqueryHasHistory || specHistExists;
+            availability.details.piRankings          = bigqueryHasPIRankings || piRankingsSnap.exists;
             
-            availability.details.piPortfolios        = piPortExists; 
-            availability.details.piDeepPortfolios    = piDeepExists; 
-            availability.details.piHistory           = piHistExists;
+            availability.details.piPortfolios        = bigqueryHasPortfolio || piPortExists; 
+            availability.details.piDeepPortfolios    = piDeepExists; // Legacy only, no BigQuery equivalent
+            availability.details.piHistory           = bigqueryHasHistory || piHistExists;
             
             // PI & Signed-In Social Flags (Strict)
-            availability.details.piSocial            = foundPISocial;
-            availability.details.hasPISocial         = foundPISocial; 
-            availability.details.signedInSocial      = foundSignedInSocial;
-            availability.details.hasSignedInSocial   = foundSignedInSocial; 
-
+            // Use BigQuery if available, otherwise use Firestore tracking documents
+            const finalPISocial = bigqueryHasSocial || foundPISocial;
+            const finalSignedInSocial = bigqueryHasSocial || foundSignedInSocial;
+            availability.details.piSocial            = finalPISocial;
+            availability.details.hasPISocial         = finalPISocial; 
+            availability.details.signedInSocial      = finalSignedInSocial;
+            availability.details.hasSignedInSocial   = finalSignedInSocial; 
+            
             // Signed-In Flags
-            availability.details.signedInUserPortfolio = signedInPortExists;
-            availability.details.signedInUserHistory   = signedInHistExists;
+            availability.details.signedInUserPortfolio = bigqueryHasPortfolio || signedInPortExists;
+            availability.details.signedInUserHistory   = bigqueryHasHistory || signedInHistExists;
             availability.details.signedInUserVerification = !verificationsQuery.empty;
             
             // New Root Data Types for Profile Metrics
@@ -461,20 +515,18 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.details.piPageViews = piPageViewsSnap.exists;
             availability.details.watchlistMembership = watchlistMembershipSnap.exists;
             availability.details.piAlertHistory = piAlertHistorySnap.exists;
+            
+            // Aggregates (use BigQuery if available, otherwise Firestore)
+            availability.hasPortfolio = bigqueryHasPortfolio || normPortExists || specPortExists || piPortExists || signedInPortExists;
+            availability.hasHistory   = bigqueryHasHistory || normHistExists || specHistExists || piHistExists || signedInHistExists;
+            availability.hasInsights  = bigqueryHasInsights || insightsSnap.exists;
+            availability.hasSocial    = bigqueryHasSocial || finalPISocial || finalSignedInSocial || genericSocialExists;
 
-            // Aggregates
-            availability.hasPortfolio = normPortExists || specPortExists || piPortExists || signedInPortExists;
-            availability.hasHistory   = normHistExists || specHistExists || piHistExists || signedInHistExists;
-            availability.hasInsights  = insightsSnap.exists;
-            availability.hasSocial    = foundPISocial || foundSignedInSocial || genericSocialExists; 
-
-            // [NEW] Assign Master List Availability
-            availability.details.piMasterList = piMasterListSnap.exists;
+            // [NEW] Assign Master List Availability (BigQuery first, then Firestore)
+            availability.details.piMasterList = bigqueryHasPIMasterList || piMasterListSnap.exists;
             
-            // Price Check
-            // Check if the target date exists in the price availability set
-            // The set is populated from the price tracking document's datesAvailable array
-            const hasPriceForDate = priceAvailabilitySet.has(dateStr);
+            // Price Check (BigQuery first, then Firestore tracking document)
+            const hasPriceForDate = bigqueryHasPrices || priceAvailabilitySet.has(dateStr);
             availability.hasPrices = hasPriceForDate;
             
             if (targetDate && !hasPriceForDate) {

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -2,13 +2,20 @@
  * @fileoverview Data Storage Helpers for New Collection Structure
  * 
  * Stores data to:
- * 1. Root data collections (date-based, per-user) - for computation system
+ * 1. BigQuery (date-based, per-user) - for computation system (NEW)
  * 2. User-centric collections (latest snapshot) - for fallback/quick access
  * 
+ * UPDATED: Removed root data Firestore writes, now writes to BigQuery instead
  * Uses the centralized collection registry for all paths.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
+const { 
+    ensurePortfolioSnapshotsTable, 
+    ensureTradeHistorySnapshotsTable, 
+    ensureSocialPostSnapshotsTable,
+    insertRows 
+} = require('../../core/utils/bigquery_utils');
 
 /**
  * Store portfolio data for a signed-in user
@@ -19,23 +26,38 @@ const { FieldValue } = require('@google-cloud/firestore');
  * @param {string} params.cid - User CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.portfolioData - Portfolio data to store
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: SignedInUserPortfolioData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('SignedInUserPortfolioData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    await rootDataRef.set({
-        ...portfolioData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    }, { merge: false });
+async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'SIGNED_IN_USER',
+                portfolio_data: JSON.stringify(portfolioData), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addPortfolioRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensurePortfolioSnapshotsTable(logger);
+                await insertRows(datasetId, 'portfolio_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for signed-in user ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
+    }
     
-    // 2. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -63,23 +85,38 @@ async function storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid,
  * @param {string} params.cid - User CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.historyData - Trade history data to store
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date, historyData }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: SignedInUserTradeHistoryData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('SignedInUserTradeHistoryData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    await rootDataRef.set({
-        ...historyData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    }, { merge: false });
+async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date, historyData, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'SIGNED_IN_USER',
+                history_data: JSON.stringify(historyData), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addHistoryRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensureTradeHistorySnapshotsTable(logger);
+                await insertRows(datasetId, 'trade_history_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for signed-in user ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
+    }
     
-    // 2. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -107,30 +144,39 @@ async function storeSignedInUserTradeHistory({ db, logger, collectionRegistry, c
  * @param {string} params.cid - User CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {Array} params.posts - Array of social posts
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date, posts }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: SignedInUserSocialPostData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('SignedInUserSocialPostData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    const postsMap = {};
-    for (const post of posts) {
-        if (post.id || post.postId) {
-            postsMap[post.id || post.postId] = post;
+async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date, posts, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            await ensureSocialPostSnapshotsTable(logger);
+            
+            const postsMap = {};
+            for (const post of posts) {
+                if (post.id || post.postId) {
+                    postsMap[post.id || post.postId] = post;
+                }
+            }
+            
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'SIGNED_IN_USER',
+                posts_data: JSON.stringify({ posts: postsMap, postCount: posts.length }), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            await insertRows(datasetId, 'social_post_snapshots', [row], logger);
+            logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for signed-in user ${cid} (date: ${date}, ${posts.length} posts)`);
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for signed-in user ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
         }
     }
     
-    await rootDataRef.set({
-        posts: postsMap,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid),
-        postCount: posts.length
-    }, { merge: false });
-    
     // 2. Store latest posts to user-centric collection (for fallback)
     // Path structure: SignedInUsers/{cid}/posts/{postId}
     // Construct path directly - we know the structure
@@ -180,31 +226,48 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.portfolioData - Portfolio data to store
  * @param {object} params.deepPortfolioData - Optional deep portfolio data
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData, deepPortfolioData = null }) {
-    // 1. Store overall portfolio to root data collection
-    // Structure: PopularInvestorPortfolioData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('PopularInvestorPortfolioData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    const portfolioDoc = {
-        ...portfolioData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    };
-    
-    // 2. If deep portfolio data exists, merge it
-    if (deepPortfolioData && deepPortfolioData.positions) {
-        portfolioDoc.deepPositions = deepPortfolioData.positions;
-        portfolioDoc.deepFetchedAt = FieldValue.serverTimestamp();
+async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date, portfolioData, deepPortfolioData = null, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const portfolioDoc = {
+                ...portfolioData,
+                cid: String(cid)
+            };
+            
+            // If deep portfolio data exists, merge it
+            if (deepPortfolioData && deepPortfolioData.positions) {
+                portfolioDoc.deepPositions = deepPortfolioData.positions;
+            }
+            
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'POPULAR_INVESTOR',
+                portfolio_data: JSON.stringify(portfolioDoc), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addPortfolioRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensurePortfolioSnapshotsTable(logger);
+                await insertRows(datasetId, 'portfolio_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote portfolio to BigQuery for PI ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
     }
     
-    await rootDataRef.set(portfolioDoc, { merge: false });
-    
-    // 3. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -231,23 +294,38 @@ async function storePopularInvestorPortfolio({ db, logger, collectionRegistry, c
  * @param {string} params.cid - PI CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {object} params.historyData - Trade history data to store
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date, historyData }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: PopularInvestorTradeHistoryData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('PopularInvestorTradeHistoryData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    await rootDataRef.set({
-        ...historyData,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid)
-    }, { merge: false });
+async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date, historyData, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'POPULAR_INVESTOR',
+                history_data: JSON.stringify(historyData), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addHistoryRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensureTradeHistorySnapshotsTable(logger);
+                await insertRows(datasetId, 'trade_history_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote trade history to BigQuery for PI ${cid} (date: ${date})`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
+        }
+    }
     
-    // 2. Store latest snapshot to user-centric collection (for fallback)
+    // 2. Store latest snapshot to user-centric collection (for fallback/quick access)
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
@@ -275,30 +353,44 @@ async function storePopularInvestorTradeHistory({ db, logger, collectionRegistry
  * @param {string} params.cid - PI CID
  * @param {string} params.date - Date string (YYYY-MM-DD)
  * @param {Array} params.posts - Array of social posts
+ * @param {object} params.bigqueryBatchManager - Optional BigQuery batch manager (if provided, batches writes)
  * @returns {Promise<void>}
  */
-async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date, posts }) {
-    // 1. Store to root data collection (for computations)
-    // Structure: PopularInvestorSocialPostData/{date}/{cid}/{cid}
-    const rootDataRef = db.collection('PopularInvestorSocialPostData')
-        .doc(date)
-        .collection(String(cid))
-        .doc(String(cid));
-    
-    const postsMap = {};
-    for (const post of posts) {
-        if (post.id || post.postId) {
-            postsMap[post.id || post.postId] = post;
+async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date, posts, bigqueryBatchManager = null }) {
+    // 1. Write to BigQuery (for computations) - use batch manager if provided, otherwise direct write
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const postsMap = {};
+            for (const post of posts) {
+                if (post.id || post.postId) {
+                    postsMap[post.id || post.postId] = post;
+                }
+            }
+            
+            const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+            const row = {
+                date: date,
+                user_id: Number(cid),
+                user_type: 'POPULAR_INVESTOR',
+                posts_data: JSON.stringify({ posts: postsMap, postCount: posts.length }), // BigQuery JSON type requires a string
+                fetched_at: new Date().toISOString()
+            };
+            
+            if (bigqueryBatchManager) {
+                // Add to batch (will flush with Firestore batches)
+                await bigqueryBatchManager.addSocialRow(row);
+            } else {
+                // Direct write (fallback for when batch manager not available)
+                await ensureSocialPostSnapshotsTable(logger);
+                await insertRows(datasetId, 'social_post_snapshots', [row], logger);
+                logger.log('INFO', `[DataStorage] ✅ Wrote social posts to BigQuery for PI ${cid} (date: ${date}, ${posts.length} posts)`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataStorage] BigQuery write failed for PI ${cid} (${date}): ${bqError.message}`);
+            // Continue to Firestore write (fallback)
         }
     }
     
-    await rootDataRef.set({
-        posts: postsMap,
-        fetchedAt: FieldValue.serverTimestamp(),
-        cid: String(cid),
-        postCount: posts.length
-    }, { merge: false });
-    
     // 2. Store latest posts to user-centric collection (for fallback)
     // Path structure: PopularInvestors/{piCid}/posts/{postId}
     // Construct path directly - we know the structure