bulltrackers-module 1.0.710 → 1.0.713

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -5,6 +5,7 @@ const { dispatchSyncRequest } = require('../task_engine_helper.js');
 const { sanitizeCid, sanitizeDocId } = require('../security_utils.js');
 const crypto = require('crypto');
 const zlib = require('zlib');
+const { query: bigqueryQuery } = require('../../../core/utils/bigquery_utils');
 
 const storage = new Storage(); // Singleton GCS Client
 
@@ -3025,16 +3026,14 @@ const getWatchlistTriggerCounts = async (db, userId, watchlistId) => {
         throw error;
     }
 };
+
 /**
  * Query PIs matching dynamic watchlist criteria over a time range.
- * * UPDATED LOGIC: 
- * Instead of only looking at the latest data snapshot, this function scans 
- * the entire requested time range (e.g. 7 days).
- * * It returns any PI who met the criteria AT LEAST ONCE during the period.
- * It provides a history of their matches, allowing the UI to show:
- * - When they first appeared (firstMatchedAt)
- * - If they are no longer matching (droppedOffAt)
- * - Their value history over the period
+ * UPDATED LOGIC: 
+ * - Scans the entire requested time range (e.g. 7 days).
+ * - Tracks history of matches vs non-matches.
+ * - droppedOffAt is now an ARRAY of dates where the user stopped matching.
+ * - Handles users disappearing from the dataset as a "drop-off" event.
  * * @param {Object} db - Firestore instance
  * @param {string} computationName - Name of the computation to query
  * @param {Object} parameters - Threshold parameters (e.g., {minChange: 1})
@@ -3061,14 +3060,38 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                 startDate.setDate(startDate.getDate() - 7);
         }
         
+        const startDateStr = startDate.toISOString().split('T')[0];
+        const endDateStr = endDate.toISOString().split('T')[0];
+        
+        // 2. Try BigQuery first (if enabled)
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const bigqueryResult = await queryDynamicWatchlistMatchesBigQuery(
+                    computationName,
+                    parameters,
+                    startDateStr,
+                    endDateStr,
+                    limit,
+                    db
+                );
+                if (bigqueryResult) {
+                    console.log(`[queryDynamicWatchlistMatches] Successfully queried from BigQuery`);
+                    return bigqueryResult;
+                }
+            } catch (bqError) {
+                console.warn(`[queryDynamicWatchlistMatches] BigQuery query failed, falling back to Firestore: ${bqError.message}`);
+                // Fall through to Firestore logic
+            }
+        }
+        
+        // 3. Fallback to Firestore (original logic)
         // Build list of dates to check (Newest -> Oldest)
         const dates = [];
         for (let d = new Date(endDate); d >= startDate; d.setDate(d.getDate() - 1)) {
             dates.push(d.toISOString().split('T')[0]);
         }
         
-        // 2. Fetch Data for ALL Dates in Parallel
-        // We do not stop at the first match anymore.
+        // 4. Fetch Data for ALL Dates in Parallel
         const datePromises = dates.map(async (dateStr) => {
             try {
                 // Try alerts path first (primary location for alert computations)
@@ -3126,44 +3149,47 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
         });
 
         // Wait for all days to load
+        // rawResults is sorted Newest -> Oldest (matches 'dates' order)
         const rawResults = (await Promise.all(datePromises)).filter(r => r !== null);
         
-        // 3. Aggregate Matches Per User
+        // 5. Aggregate Matches Per User
         // Map: piCid -> { firstMatchedAt, lastMatchedAt, history: [], ... }
         const piAggregates = new Map();
         
         // Process dates from Oldest -> Newest to build timeline correctly
-        // (rawResults is Newest->Oldest, so we reverse it)
-        const timeline = rawResults.reverse(); 
-
+        // Note: .reverse() mutates the array in place, so rawResults becomes Oldest->Newest
+        const timeline = rawResults.reverse();
+        
         for (const dayEntry of timeline) {
             const { date, data } = dayEntry;
-
+            const seenCidsThisDay = new Set();
+            
+            // A. Process Users Present in the Daily File
             for (const [piCidStr, piData] of Object.entries(data)) {
                 if (piData.error) continue;
-
+                
                 const piCid = Number(piCidStr);
+                seenCidsThisDay.add(piCid);
                 
-                // Check if this PI matches the criteria ON THIS SPECIFIC DAY
                 const filterResult = checkPIMatchesCriteria(computationName, piData, parameters);
-
+                
                 if (filterResult.passes) {
+                    // Initialize if new
                     if (!piAggregates.has(piCid)) {
                         piAggregates.set(piCid, {
                             cid: piCid,
-                            firstMatchedAt: date, // Earliest date in range they matched
-                            lastMatchedAt: date,  // Will update as we go forward
-                            droppedOffAt: null,   // Will calculate later
+                            firstMatchedAt: date,
+                            lastMatchedAt: date,
                             matchCount: 0,
-                            history: [],          // Array of { date, value, change }
-                            latestData: null      // Store latest raw data for metadata
+                            history: [],
+                            latestData: null
                         });
                     }
-
+                    
                     const agg = piAggregates.get(piCid);
                     agg.lastMatchedAt = date;
                     agg.matchCount++;
-                    agg.latestData = piData; // Update to keep the most recent data packet
+                    agg.latestData = piData;
                     
                     agg.history.push({
                         date: date,
@@ -3172,42 +3198,70 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                         change: filterResult.change
                     });
                 } else {
-                    // PI exists in data but DOES NOT match criteria (e.g. Risk score dropped back down)
+                    // User exists in data but DOES NOT match criteria
                     if (piAggregates.has(piCid)) {
                         const agg = piAggregates.get(piCid);
-                        // Record the non-match event in history if they had previously matched
+                        // Update metadata to show why they failed (current value)
+                        agg.latestData = piData;
+                        
                         agg.history.push({
                             date: date,
                             matched: false,
-                            value: filterResult.matchValue, // Value that failed criteria
+                            value: filterResult.matchValue,
                             change: filterResult.change
                         });
                     }
                 }
             }
+            
+            // B. Process Missing Users (Implicit Drop-off)
+            // If a user was tracked previously but is missing today, record as non-match
+            for (const [cid, agg] of piAggregates) {
+                if (!seenCidsThisDay.has(cid)) {
+                    agg.history.push({
+                        date: date,
+                        matched: false,
+                        value: null, // Value unknown/missing
+                        change: null
+                    });
+                }
+            }
         }
-
-        // 4. Calculate Status (Dropped Off, Current) & Fetch Usernames
+        
+        // 6. Calculate Status (Dropped Off, Current) & Fetch Usernames
         const results = [];
         const todayStr = new Date().toISOString().split('T')[0];
-        const lastDataDate = rawResults.length > 0 ? rawResults[rawResults.length - 1].date : todayStr; // The "Current" date of the system
-
+        // Since rawResults was reversed, the last element is the Newest date
+        const lastDataDate = timeline.length > 0 ? timeline[timeline.length - 1].date : todayStr;
+        
         for (const [cid, agg] of piAggregates) {
-            // Determine if "Dropped Off"
-            // They dropped off if their last match was BEFORE the most recent data date we have
-            let isCurrent = agg.lastMatchedAt === lastDataDate;
-            let droppedOffAt = isCurrent ? null : agg.history[agg.history.length - 1]?.date;
-
-            // Fetch Username (Optimistic/Lazy)
+            const history = agg.history;
+            const lastEntry = history[history.length - 1];
+            
+            // Is Currently Matching?
+            // Must be matched=true AND on the most recent data date available
+            const isCurrent = lastEntry.matched && lastEntry.date === lastDataDate;
+            
+            // Calculate Drop Off Dates
+            // Find all transitions from True -> False
+            const droppedOffAt = [];
+            for (let i = 1; i < history.length; i++) {
+                const prev = history[i - 1];
+                const curr = history[i];
+                if (prev.matched && !curr.matched) {
+                    droppedOffAt.push(curr.date);
+                }
+            }
+            
+            // Fetch Username (Optimistic)
             let username = `PI-${cid}`;
-            try {
-                // Assuming this is cached or fast enough. For 1500 users this might be slow loop.
-                // In production, better to fetch all usernames in one batch or store in the computation data.
-                // Falling back to "Unknown" if costly, or rely on frontend to fetch details.
-                const piProfile = await fetchPopularInvestorMasterList(db, String(cid)).catch(() => null);
-                if (piProfile) username = piProfile.username;
-            } catch (e) {}
-
+            if (db) {
+                try {
+                    const piProfile = await fetchPopularInvestorMasterList(db, String(cid)).catch(() => null);
+                    if (piProfile) username = piProfile.username;
+                } catch (e) {}
+            }
+            
             results.push({
                 cid: cid,
                 username: username,
@@ -3215,50 +3269,52 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                 // Aggregated Stats
                 firstMatchedAt: agg.firstMatchedAt,
                 lastMatchedAt: agg.lastMatchedAt,
-                droppedOffAt: droppedOffAt, // If set, they matched previously but don't match now
+                
+                // [UPDATED] Array of dates where they stopped matching
+                droppedOffAt: droppedOffAt,
+                
                 isCurrentlyMatching: isCurrent,
                 matchCount: agg.matchCount,
                 
                 // Visualization Data
-                history: agg.history, 
+                history: agg.history,
                 
-                // Latest Snapshot Values (for sorting/display)
-                latestValue: agg.history[agg.history.length - 1]?.value,
+                // Latest Snapshot Values
+                latestValue: history[history.length - 1]?.value,
                 
-                // Metadata for debug/details
+                // Metadata
                 metadata: agg.latestData
             });
         }
-
-        // 5. Sort Results
+        
+        // 7. Sort Results
         // Priority: Currently Matching > Recently Dropped Off
-        // Secondary: Match Value (High Risk/High Change first)
+        // Secondary: Match Value magnitude
         results.sort((a, b) => {
             if (a.isCurrentlyMatching !== b.isCurrentlyMatching) {
                 return a.isCurrentlyMatching ? -1 : 1;
             }
-            // If both current or both dropped, sort by latest value magnitude
-            return Math.abs(b.latestValue) - Math.abs(a.latestValue);
+            // If both same status, sort by magnitude of value (risk, change, etc)
+            return Math.abs(b.latestValue || 0) - Math.abs(a.latestValue || 0);
         });
-
+        
         const limitedResults = results.slice(0, limit);
-
+        
         console.log(`[queryDynamicWatchlistMatches] Found ${results.length} unique PIs matching at least once.`);
-
+        
         return {
             success: true,
             matches: limitedResults,
             count: limitedResults.length,
             totalUniqueMatches: results.length,
             dateRange: {
-                start: startDate.toISOString().split('T')[0],
-                end: endDate.toISOString().split('T')[0]
+                start: startDateStr,
+                end: endDateStr
             },
-            dataDate: lastDataDate, // Most recent date found in system
+            dataDate: lastDataDate,
             computationName,
             parameters
         };
-
     } catch (error) {
         console.error(`[queryDynamicWatchlistMatches] Error: ${error.message}`, error);
         throw error;

package/functions/computation-system/data/CachedDataLoader.js

@@ -189,7 +189,28 @@ class CachedDataLoader {
         return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); 
     }
 
-    async loadPriceShard(docRef) { 
+    async loadPriceShard(docRef) {
+        // Check if this is a BigQuery marker
+        if (docRef && docRef._bigquery === true) {
+            // Load all price data from BigQuery
+            try {
+                const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
+                const priceData = await queryAssetPrices(null, null, null, this.deps.logger);
+                
+                if (priceData && Object.keys(priceData).length > 0) {
+                    this.deps.logger.log('INFO', `[CachedDataLoader] ✅ Loaded ${Object.keys(priceData).length} instruments from BigQuery`);
+                    return priceData;
+                }
+                
+                // If BigQuery returns empty, fallback to Firestore
+                this.deps.logger.log('WARN', `[CachedDataLoader] BigQuery returned no price data, falling back to Firestore`);
+            } catch (bqError) {
+                this.deps.logger.log('WARN', `[CachedDataLoader] BigQuery price load failed, falling back to Firestore: ${bqError.message}`);
+                // Fall through to Firestore
+            }
+        }
+        
+        // Firestore fallback (original logic)
         try { 
             const snap = await docRef.get(); 
             return snap.exists ? this._tryDecompress(snap.data()) : {}; 

package/functions/computation-system/data/DependencyFetcher.js

@@ -44,11 +44,36 @@ function tryDecompress(payload) {
 
 /**
  * Fetches, decompresses, and reassembles (if sharded or on GCS) a single result document.
+ * NEW: For non-alert, non-page computations, tries BigQuery first (cheaper, no sharding/compression).
  */
 async function fetchSingleResult(db, config, dateStr, name, category) {
     const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
     const log = config.logger || console;
 
+    // NEW STRATEGY: Check if this is an alert or page computation
+    // We need to check the manifest to determine this, but we can infer from category
+    // For now, we'll try BigQuery first for all non-alert computations (alerts are in 'alerts' category)
+    const isAlertComputation = category === 'alerts';
+    // Page computations are typically in 'popular-investor' category but have isPage flag
+    // For now, we'll try BigQuery for all non-alert computations
+    
+    // Try BigQuery first for non-alert computations (reduces Firestore reads)
+    if (!isAlertComputation && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryComputationResult } = require('../../core/utils/bigquery_utils');
+            const bigqueryResult = await queryComputationResult(name, category, dateStr, log);
+            
+            if (bigqueryResult && !isDataEmpty(bigqueryResult)) {
+                log.log('INFO', `[DependencyFetcher] ✅ Using BigQuery for ${name} (${dateStr}, ${category})`);
+                return bigqueryResult;
+            }
+        } catch (bqError) {
+            log.log('WARN', `[DependencyFetcher] BigQuery fetch failed for ${name}, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // Fallback to Firestore (for alerts, pages, or if BigQuery fails)
     const docRef = db.collection(resultsCollection).doc(dateStr)
                      .collection(resultsSubcollection).doc(category)
                      .collection(computationsSubcollection).doc(name);
@@ -238,6 +263,9 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
         d.setUTCDate(d.getUTCDate() - 1);
         dates.push(d.toISOString().slice(0, 10));
     }
+    
+    const startDateStr = dates[dates.length - 1]; // Oldest date
+    const queryEndDateStr = dates[0]; // Newest date (for BigQuery query)
 
     // [DEBUG] Log the manifest lookup and resolved categories
     logger.log('INFO', `[DependencyFetcher] 🔍 ManifestLookup has ${Object.keys(manifestLookup).length} entries`);
@@ -248,6 +276,96 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
         logger.log('INFO', `[DependencyFetcher] 📍 '${rawName}' -> category='${category}' -> Path: ${samplePath}`);
     }
 
+    // =========================================================================
+    // BIGQUERY FIRST: Try batch query for all dates at once
+    // =========================================================================
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryComputationResultsRange } = require('../../core/utils/bigquery_utils');
+            
+            // Query each computation in parallel
+            const bigqueryPromises = calcNames.map(async (rawName) => {
+                const norm = normalizeName(rawName);
+                const category = manifestLookup[norm] || 'analytics';
+                
+                const bigqueryRows = await queryComputationResultsRange(
+                    rawName,
+                    category,
+                    startDateStr,
+                    queryEndDateStr,
+                    logger
+                );
+                
+                if (bigqueryRows && bigqueryRows.length > 0) {
+                    logger.log('INFO', `[DependencyFetcher] ✅ Using BigQuery for ${rawName} series: ${bigqueryRows.length} dates`);
+                    
+                    // Map BigQuery results to results structure
+                    for (const row of bigqueryRows) {
+                        if (row.data && !isDataEmpty(row.data)) {
+                            results[norm][row.date] = row.data;
+                        }
+                    }
+                    
+                    return { name: rawName, found: bigqueryRows.length };
+                }
+                
+                return { name: rawName, found: 0 };
+            });
+            
+            const bigqueryResults = await Promise.all(bigqueryPromises);
+            const totalFound = bigqueryResults.reduce((sum, r) => sum + r.found, 0);
+            
+            if (totalFound > 0) {
+                logger.log('INFO', `[DependencyFetcher] ✅ BigQuery retrieved ${totalFound} computation result records across ${calcNames.length} computations`);
+                
+                // Fill in any missing dates from Firestore (fallback)
+                const missingOps = [];
+                for (const dateStr of dates) {
+                    for (const rawName of calcNames) {
+                        const norm = normalizeName(rawName);
+                        // Only fetch if we don't have this date already
+                        if (!results[norm] || !results[norm][dateStr]) {
+                            const category = manifestLookup[norm] || 'analytics';
+                            missingOps.push(async () => {
+                                const val = await fetchSingleResult(db, { ...config, logger }, dateStr, rawName, category);
+                                if (val && !isDataEmpty(val)) {
+                                    results[norm][dateStr] = val;
+                                }
+                            });
+                        }
+                    }
+                }
+                
+                // Fetch missing dates from Firestore
+                if (missingOps.length > 0) {
+                    logger.log('INFO', `[DependencyFetcher] 📂 Fetching ${missingOps.length} missing dates from Firestore (fallback)`);
+                    const BATCH_SIZE = 20;
+                    for (let i = 0; i < missingOps.length; i += BATCH_SIZE) {
+                        await Promise.all(missingOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
+                    }
+                }
+                
+                // Log final summary
+                for (const rawName of calcNames) {
+                    const norm = normalizeName(rawName);
+                    const foundDates = Object.keys(results[norm] || {});
+                    logger.log('INFO', `[DependencyFetcher] ✅ '${rawName}' found data for ${foundDates.length}/${lookbackDays} days (BigQuery + Firestore)`);
+                }
+                
+                return results;
+            } else {
+                logger.log('INFO', `[DependencyFetcher] ⚠️ BigQuery returned no results, falling back to Firestore`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DependencyFetcher] BigQuery series query failed, falling back to Firestore: ${bqError.message}`);
+        }
+    }
+    
+    // =========================================================================
+    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
+    // =========================================================================
+    logger.log('INFO', `[DependencyFetcher] 📂 Using Firestore for computation result series: ${calcNames.length} calcs x ${lookbackDays} days`);
+
     // Build Fetch Operations
     const ops = [];
     for (const dateStr of dates) {

package/functions/computation-system/persistence/ResultCommitter.js

@@ -16,6 +16,7 @@ const validationOverrides                  = require('../config/validation_overr
 const pLimit                               = require('p-limit'); 
 const zlib                                 = require('zlib'); 
 const { Storage }                          = require('@google-cloud/storage');
+const { ensureComputationResultsTable, insertRows } = require('../../core/utils/bigquery_utils');
 
 const storage = new Storage(); // Singleton GCS Client
 const NON_RETRYABLE_ERRORS = [ 'PERMISSION_DENIED', 'DATA_LOSS', 'FAILED_PRECONDITION' ];
@@ -210,7 +211,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     const historicalDocRef = db.collection(config.resultsCollection).doc(historicalDate).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
                     
                     // Recursive call allows GCS logic to apply per-day
-                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, calc.manifest.category, logger, config, deps, 0, 'STANDARD', false, dailyExpireAt);
+                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, calc.manifest.category, logger, config, deps, 0, 'STANDARD', false, dailyExpireAt, isAlertComputation, isPageComputation);
                     runMetrics.io.writes += stats.opCounts.writes;
                     runMetrics.io.deletes += stats.opCounts.deletes;
 
@@ -225,7 +226,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 const runExpireAt = calculateExpirationDate(dStr, ttlDays);
 
                 const mainDocRef = db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, calc.manifest.category, logger, config, deps, currentShardIndex, flushMode, isInitialWrite, runExpireAt);
+                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, calc.manifest.category, logger, config, deps, currentShardIndex, flushMode, isInitialWrite, runExpireAt, isAlertComputation, isPageComputation);
                 
                 runMetrics.storage.sizeBytes  = writeStats.totalSize;
                 runMetrics.storage.isSharded  = writeStats.isSharded;
@@ -294,7 +295,7 @@ async function fetchContracts(db, calcNames) {
     return map;
 }
 
-async function writeSingleResult(result, docRef, name, dateContext, category, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false, expireAt = null) {
+async function writeSingleResult(result, docRef, name, dateContext, category, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false, expireAt = null, isAlertComputation = false) {
     const opCounts = { writes: 0, deletes: 0 };
     
     // Check if previously sharded (so we can clean up if moving to GCS or Compressed)
@@ -358,6 +359,13 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
 
             logger.log('INFO', `[GCS] ${name}: Offloaded ${(totalSize/1024).toFixed(0)}KB to ${fileName}`);
             
+            // Write to BigQuery (await to ensure completion before function returns)
+            // Errors are caught and logged but don't fail the operation
+            // Pass isAlertComputation flag to use streaming for alerts, load jobs for others
+            await writeToBigQuery(result, name, dateContext, category, logger, isAlertComputation).catch(err => {
+                logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
+            });
+            
             return { totalSize, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts, location: 'GCS' };
 
         } catch (gcsErr) {
@@ -394,6 +402,12 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
                 opCounts.writes += 1;
                 logger.log('INFO', `[Compression] ${name}: Compressed ${(totalSize/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB.`);
                 
+                // Write to BigQuery (await to ensure completion before function returns)
+                // Errors are caught and logged but don't fail the operation
+                await writeToBigQuery(result, name, dateContext, category, logger).catch(err => {
+                    logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
+                });
+                
                 return { totalSize: compressedBuffer.length, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts, location: 'FIRESTORE' };
             }
         } catch (compErr) {
@@ -475,6 +489,13 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
         shardingError.stage = 'SHARDING_LIMIT_EXCEEDED';
         throw shardingError; 
     }
+    
+    // Write to BigQuery (await to ensure completion before function returns)
+    // Errors are caught and logged but don't fail the operation
+    await writeToBigQuery(result, name, dateContext, category, logger).catch(err => {
+        logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
+    });
+    
     finalStats.opCounts = opCounts;
     return finalStats;
 }
@@ -483,6 +504,76 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
 // HELPERS
 // =============================================================================
 
+/**
+ * Write computation result to BigQuery (errors are logged but don't fail Firestore writes)
+ * @param {object} result - Computation result data
+ * @param {string} name - Computation name
+ * @param {string} dateContext - Date string (YYYY-MM-DD)
+ * @param {string} category - Category (e.g., 'popular-investor', 'alerts')
+ * @param {object} logger - Logger instance
+ * @param {boolean} isAlertComputation - If true, uses streaming inserts (immediate). If false, uses load jobs (batched, free).
+ */
+async function writeToBigQuery(result, name, dateContext, category, logger, isAlertComputation = false) {
+    // Skip if BigQuery is disabled via environment variable
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        return;
+    }
+    
+    try {
+        // Size check: BigQuery streaming inserts have a 10MB limit per row
+        // Estimate size by stringifying the result
+        const estimatedSize = JSON.stringify(result).length;
+        const MAX_BIGQUERY_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit (10MB is hard limit)
+        
+        if (estimatedSize > MAX_BIGQUERY_ROW_SIZE) {
+            if (logger) {
+                logger.log('WARN', `[BigQuery] Skipping ${name} (${dateContext}): Result too large for streaming (${(estimatedSize/1024/1024).toFixed(2)}MB). Data is in GCS/Firestore.`);
+            }
+            // Return early - don't attempt insert that will fail
+            // The data is still available in Firestore/GCS, so this is acceptable
+            return;
+        }
+        
+        // Ensure table exists
+        await ensureComputationResultsTable(logger);
+        
+        // Extract metadata (cids if present)
+        const metadata = {};
+        if (result.cids && Array.isArray(result.cids)) {
+            metadata.cids = result.cids;
+        }
+        
+        // Prepare row for BigQuery
+        const row = {
+            date: dateContext,
+            computation_name: name,
+            category: category,
+            result_data: result, // Full result as JSON
+            metadata: Object.keys(metadata).length > 0 ? metadata : null,
+            created_at: new Date().toISOString()
+        };
+        
+        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        
+        // Use streaming inserts for alert computations (immediate, time-sensitive)
+        // Use load jobs for non-alert computations (batched, free)
+        const { insertRows: insertRowsLoadJob, insertRowsStreaming } = require('../../core/utils/bigquery_utils');
+        
+        if (isAlertComputation) {
+            await insertRowsStreaming(datasetId, 'computation_results', [row], logger);
+        } else {
+            await insertRowsLoadJob(datasetId, 'computation_results', [row], logger);
+        }
+        
+    } catch (error) {
+        // Log but don't throw - BigQuery write failure shouldn't break Firestore writes
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Write failed for ${name} (${dateContext}): ${error.message}`);
+        }
+        // Don't re-throw - ensure Firestore writes always succeed
+    }
+}
+
 async function cleanupOldShards(docRef, name, config, deps, metrics) {
     const shardCol = docRef.collection('_shards');
     const shardDocs = await shardCol.listDocuments();