bulltrackers-module 1.0.721 → 1.0.723

package/functions/core/utils/bigquery_utils.js

@@ -100,166 +100,94 @@ async function ensureTableExists(datasetId, tableId, schema, options = {}, logge
         throw error;
     }
 }
-
 /**
  * Insert rows using BigQuery MERGE statement (handles duplicates natively via SQL)
- * More efficient than checking then inserting - BigQuery handles deduplication in SQL
- * Uses a temporary table and MERGE statement for atomic deduplication
- * @param {string} datasetId - Dataset ID
- * @param {string} tableId - Table ID
- * @param {Array} rows - Array of row objects
- * @param {Array} keyFields - Fields that form unique key (e.g., ['date', 'user_id', 'user_type'])
- * @param {object} logger - Logger instance
- * @returns {Promise<number>} Number of rows actually inserted (not duplicates)
+ * FIXED: Ensures temporary tables are always deleted using try/finally block.
  */
 async function insertRowsWithMerge(datasetId, tableId, rows, keyFields, logger = null) {
-    if (!rows || rows.length === 0) {
-        if (logger) logger.log('WARN', `[BigQuery] No rows to merge into ${datasetId}.${tableId}`);
-        return 0;
-    }
+    if (!rows || rows.length === 0) return 0;
     
-    const MAX_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit
-    const validRows = rows.filter(row => {
-        const rowSize = JSON.stringify(row).length;
-        return rowSize <= MAX_ROW_SIZE;
-    });
+    const MAX_ROW_SIZE = 9 * 1024 * 1024; 
+    const validRows = rows.filter(row => JSON.stringify(row).length <= MAX_ROW_SIZE);
     
     if (validRows.length === 0) {
         if (logger) logger.log('WARN', `[BigQuery] All rows too large for MERGE into ${datasetId}.${tableId}`);
         return 0;
     }
+
+    // 1. Define ID once, outside the try block, so it's consistent in finally/catch
+    const tempTableId = `${tableId}_temp_${Date.now()}_${Math.random().toString(36).substring(7)}`;
+    const tablePath = `${datasetId}.${tableId}`;
+    const tempTablePath = `${datasetId}.${tempTableId}`;
     
     try {
-        const tablePath = `${datasetId}.${tableId}`;
-        const keyFieldsStr = keyFields.join(', ');
-        const tempTableId = `${tableId}_temp_${Date.now()}`;
-        const tempTablePath = `${datasetId}.${tempTableId}`;
-        
-        // Get table schema
         const dataset = await getOrCreateDataset(datasetId, logger);
         const table = dataset.table(tableId);
         const [tableMetadata] = await table.getMetadata();
         const schema = tableMetadata.schema.fields;
         
-        // Create temp table with same schema
+        // 2. Create Temp Table
         const tempTable = dataset.table(tempTableId);
-        await tempTable.create({
-            schema: schema,
-            description: 'Temporary table for merge operation'
+        await tempTable.create({ 
+            schema: schema, 
+            expirationTime: Date.now() + (1000 * 60 * 60 * 1), // Auto-delete after 1 hour (safety net)
+            description: 'Temporary table for merge operation' 
         });
         
-        if (logger) {
-            logger.log('INFO', `[BigQuery] Created temp table ${tempTableId} for MERGE operation`);
-        }
-        
-        // Insert all rows into temp table using LOAD JOB (free, not streaming)
-        // Write to temporary file (load jobs require a file, not a stream)
-        const tempFile = path.join(os.tmpdir(), `bigquery_merge_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
+        // 3. Load Data into Temp Table
+        const tempFile = path.join(os.tmpdir(), `bq_merge_${tempTableId}.ndjson`);
         const ndjson = validRows.map(r => JSON.stringify(r)).join('\n');
         
         try {
             fs.writeFileSync(tempFile, ndjson, 'utf8');
-            
-            // Load into temp table using load job (FREE) from temp file
-            // Use createLoadJob to get a Job object we can wait on
             const [loadJob] = await tempTable.createLoadJob(tempFile, {
                 sourceFormat: 'NEWLINE_DELIMITED_JSON',
                 writeDisposition: 'WRITE_APPEND',
-                autodetect: false // Use existing table schema
+                autodetect: false
             });
-            
-            // [FIX] Use native job.promise() instead of custom polling
-            // This automatically polls and waits for completion
             await loadJob.promise();
-            
-            // Get job metadata to check for errors
-            const [jobMetadata] = await loadJob.getMetadata();
-            
-            // Check for errors
-            if (jobMetadata.status?.errorResult) {
-                throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
-            }
         } finally {
-            // Clean up temp file
-            try {
-                if (fs.existsSync(tempFile)) {
-                    fs.unlinkSync(tempFile);
-                }
-            } catch (cleanupError) {
-                if (logger) {
-                    logger.log('WARN', `[BigQuery] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
-                }
-            }
-        }
-        
-        if (logger) {
-            logger.log('INFO', `[BigQuery] Loaded ${validRows.length} rows into temp table ${tempTableId} using LOAD JOB (free)`);
+            if (fs.existsSync(tempFile)) try { fs.unlinkSync(tempFile); } catch (e) {}
         }
         
-        // Use MERGE to insert new rows or update existing rows (SQL-native deduplication/overwrite)
-        // This is more efficient than checking in JavaScript
+        // 4. Perform MERGE
         const mergeConditions = keyFields.map(f => `target.${f} = source.${f}`).join(' AND ');
-        
-        // Build UPDATE clause - update all non-key fields
         const allFields = schema.map(f => f.name);
         const nonKeyFields = allFields.filter(f => !keyFields.includes(f));
         const updateClause = nonKeyFields.map(f => `${f} = source.${f}`).join(', ');
         
-        // Count rows that will be inserted (don't exist in target) vs updated (already exist)
-        // Query BEFORE the MERGE to get accurate counts
-        const [insertedCountResult] = await query(`
+        // Query to count inserts vs updates
+        const [countRes] = await query(`
             SELECT COUNT(*) as inserted
             FROM \`${tempTablePath}\` AS source
-            WHERE NOT EXISTS (
-                SELECT 1 FROM \`${tablePath}\` AS target
-                WHERE ${mergeConditions}
-            )
+            WHERE NOT EXISTS (SELECT 1 FROM \`${tablePath}\` AS target WHERE ${mergeConditions})
         `, {}, logger);
-        
-        const rowsInserted = insertedCountResult[0]?.inserted || 0;
-        const rowsUpdated = validRows.length - rowsInserted;
-        
-        // Now perform the MERGE (inserts new rows, updates existing rows)
+        const rowsInserted = countRes[0]?.inserted || 0;
+
         const mergeQuery = `
             MERGE \`${tablePath}\` AS target
             USING \`${tempTablePath}\` AS source
             ON ${mergeConditions}
-            WHEN MATCHED THEN
-                UPDATE SET ${updateClause}
-            WHEN NOT MATCHED THEN
-                INSERT ROW
+            WHEN MATCHED THEN UPDATE SET ${updateClause}
+            WHEN NOT MATCHED THEN INSERT ROW
         `;
-        
         await query(mergeQuery, {}, logger);
         
-        // Drop temp table
-        await tempTable.delete();
-        
-        if (logger) {
-            if (rowsUpdated > 0) {
-                logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted, ${rowsUpdated} existing rows updated in ${tablePath}`);
-            } else {
-                logger.log('INFO', `[BigQuery] MERGE completed: ${rowsInserted} new rows inserted into ${tablePath} (${validRows.length - rowsInserted} duplicates skipped via SQL)`);
-            }
-        }
+        if (logger) logger.log('INFO', `[BigQuery] MERGE Success: ${rowsInserted} new, ${validRows.length - rowsInserted} updated.`);
         
         return rowsInserted;
+
     } catch (error) {
-        const errorDetails = {
-            message: error.message,
-            code: error.code,
-            errors: error.errors
-        };
-        if (logger) {
-            logger.log('ERROR', `[BigQuery] MERGE failed for ${datasetId}.${tableId}:`, JSON.stringify(errorDetails, null, 2));
-        }
-        // Try to clean up temp table if it exists
+        if (logger) logger.log('ERROR', `[BigQuery] MERGE failed for ${datasetId}.${tableId}`, error);
+        throw error;
+    } finally {
+        // 5. CLEANUP: Always delete the temp table (the previous bug was here)
         try {
             const dataset = await getOrCreateDataset(datasetId, logger);
-            const tempTableId = `${tableId}_temp_${Date.now()}`;
-            await dataset.table(tempTableId).delete().catch(() => {}); // Ignore cleanup errors
-        } catch {}
-        throw error;
+            await dataset.table(tempTableId).delete({ ignoreNotFound: true }).catch(() => {});
+        } catch (cleanupError) {
+            if (logger) logger.log('WARN', `[BigQuery] Cleanup failed for ${tempTableId}: ${cleanupError.message}`);
+        }
     }
 }
 
@@ -1700,12 +1628,31 @@ async function queryPIRankings(dateStr, logger = null) {
  * @param {object} logger - Logger instance
  * @returns {Promise<object>} Price data map in format { instrumentId: { "YYYY-MM-DD": price } }
  */
+/**
+ * Query asset prices from BigQuery with smart routing.
+ * This function intelligently routes to more efficient queries when possible:
+ * - If querying a single date with no instrument filter -> uses queryAllPricesForDate
+ * - Otherwise uses the general date range query
+ * 
+ * Returns data in format: { instrumentId: { "YYYY-MM-DD": { price, ticker, ... } } }
+ * @param {string|null} startDateStr - Start date in YYYY-MM-DD format (optional)
+ * @param {string|null} endDateStr - End date in YYYY-MM-DD format (optional)
+ * @param {number[]|null} instrumentIds - Array of instrument IDs to filter (optional)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Price map, or null if not found/error
+ */
 async function queryAssetPrices(startDateStr = null, endDateStr = null, instrumentIds = null, logger = null) {
     if (process.env.BIGQUERY_ENABLED === 'false') {
         if (logger) logger.log('DEBUG', '[BigQuery] Asset prices query skipped (BIGQUERY_ENABLED=false)');
         return null;
     }
     
+    // SMART ROUTING: If querying single date with no instrument filter, use optimized function
+    if (startDateStr && startDateStr === endDateStr && (!instrumentIds || instrumentIds.length === 0)) {
+        if (logger) logger.log('INFO', `[BigQuery] 🎯 Routing to optimized single-date query for ${startDateStr}`);
+        return queryAllPricesForDate(startDateStr, logger);
+    }
+    
     const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
     const tablePath = `${datasetId}.asset_prices`;
     
@@ -1733,7 +1680,12 @@ async function queryAssetPrices(startDateStr = null, endDateStr = null, instrume
                 instrument_id,
                 date,
                 price,
-                ticker
+                ticker,
+                open,
+                high,
+                low,
+                close,
+                volume
             FROM \`${tablePath}\`
             ${whereClause}
             ORDER BY instrument_id, date DESC
@@ -1757,7 +1709,7 @@ async function queryAssetPrices(startDateStr = null, endDateStr = null, instrume
             return {};
         }
         
-        // Transform to expected format: { instrumentId: { "YYYY-MM-DD": price } }
+        // Transform to expected format: { instrumentId: { "YYYY-MM-DD": { price, ticker, ... } } }
         const priceMap = {};
         for (const row of rows) {
             const instrumentId = String(row.instrument_id);
@@ -1767,8 +1719,16 @@ async function queryAssetPrices(startDateStr = null, endDateStr = null, instrume
                 priceMap[instrumentId] = {};
             }
             
-            // Use close price if available, otherwise use price
-            priceMap[instrumentId][dateStr] = row.price || null;
+            // Return full price data structure for consistency
+            priceMap[instrumentId][dateStr] = {
+                price: row.price || null,
+                ticker: row.ticker || null,
+                open: row.open || null,
+                high: row.high || null,
+                low: row.low || null,
+                close: row.close || null,
+                volume: row.volume || null
+            };
         }
         
         if (logger) {
@@ -1784,6 +1744,180 @@ async function queryAssetPrices(startDateStr = null, endDateStr = null, instrume
     }
 }
 
+/**
+ * Query all prices for a specific date (all tickers).
+ * Returns data in format: { instrumentId: { "YYYY-MM-DD": price } }
+ * @param {string} dateStr - Date in YYYY-MM-DD format
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Price map, or null if not found/error
+ */
+async function queryAllPricesForDate(dateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] All prices query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.asset_prices`;
+    
+    try {
+        const sqlQuery = `
+            SELECT 
+                instrument_id,
+                date,
+                price,
+                ticker,
+                open,
+                high,
+                low,
+                close,
+                volume
+            FROM \`${tablePath}\`
+            WHERE date = @date
+            ORDER BY instrument_id ASC
+        `;
+        
+        const params = { date: dateStr };
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying all prices for date ${dateStr} from ${tablePath}`);
+        }
+        
+        const rows = await query(sqlQuery, { params }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No prices found for date ${dateStr} in ${tablePath}`);
+            return {};
+        }
+        
+        // Transform to expected format: { instrumentId: { "YYYY-MM-DD": { price, ticker, ... } } }
+        const priceMap = {};
+        for (const row of rows) {
+            const instrumentId = String(row.instrument_id);
+            const dateKey = row.date; // Already in YYYY-MM-DD format
+            
+            if (!priceMap[instrumentId]) {
+                priceMap[instrumentId] = {};
+            }
+            
+            priceMap[instrumentId][dateKey] = {
+                price: row.price || null,
+                ticker: row.ticker || null,
+                open: row.open || null,
+                high: row.high || null,
+                low: row.low || null,
+                close: row.close || null,
+                volume: row.volume || null
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved prices for ${Object.keys(priceMap).length} instruments for date ${dateStr}`);
+        }
+        
+        return priceMap;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] All prices query failed for date ${dateStr}: ${error.message}`);
+        }
+        return null;
+    }
+}
+
+/**
+ * Query prices for specific ticker(s) over a date range.
+ * Returns data in format: { instrumentId: { "YYYY-MM-DD": { price, ticker, ... } } }
+ * @param {string|string[]} tickers - Single ticker string or array of ticker strings (e.g., "AAPL" or ["AAPL", "MSFT"])
+ * @param {string} startDateStr - Start date in YYYY-MM-DD format
+ * @param {string} endDateStr - End date in YYYY-MM-DD format (inclusive)
+ * @param {object} logger - Logger instance
+ * @returns {Promise<object|null>} Price map, or null if not found/error
+ */
+async function queryPricesForTickers(tickers, startDateStr, endDateStr, logger = null) {
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        if (logger) logger.log('DEBUG', '[BigQuery] Ticker prices query skipped (BIGQUERY_ENABLED=false)');
+        return null;
+    }
+    
+    if (!tickers || (Array.isArray(tickers) && tickers.length === 0)) {
+        if (logger) logger.log('WARN', '[BigQuery] No tickers provided to queryPricesForTickers');
+        return {};
+    }
+    
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    const tablePath = `${datasetId}.asset_prices`;
+    
+    try {
+        const tickerArray = Array.isArray(tickers) ? tickers : [tickers];
+        
+        const sqlQuery = `
+            SELECT 
+                instrument_id,
+                date,
+                price,
+                ticker,
+                open,
+                high,
+                low,
+                close,
+                volume
+            FROM \`${tablePath}\`
+            WHERE ticker IN UNNEST(@tickers)
+                AND date BETWEEN @startDate AND @endDate
+            ORDER BY instrument_id ASC, date DESC
+        `;
+        
+        const params = {
+            tickers: tickerArray,
+            startDate: startDateStr,
+            endDate: endDateStr
+        };
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] 🔍 Querying prices for tickers [${tickerArray.join(', ')}] from ${startDateStr} to ${endDateStr} from ${tablePath}`);
+        }
+        
+        const rows = await query(sqlQuery, { params }, logger);
+        
+        if (!rows || rows.length === 0) {
+            if (logger) logger.log('INFO', `[BigQuery] No prices found for tickers [${tickerArray.join(', ')}] in date range`);
+            return {};
+        }
+        
+        // Transform to expected format: { instrumentId: { "YYYY-MM-DD": { price, ticker, ... } } }
+        const priceMap = {};
+        for (const row of rows) {
+            const instrumentId = String(row.instrument_id);
+            const dateKey = row.date;
+            
+            if (!priceMap[instrumentId]) {
+                priceMap[instrumentId] = {};
+            }
+            
+            priceMap[instrumentId][dateKey] = {
+                price: row.price || null,
+                ticker: row.ticker || null,
+                open: row.open || null,
+                high: row.high || null,
+                low: row.low || null,
+                close: row.close || null,
+                volume: row.volume || null
+            };
+        }
+        
+        if (logger) {
+            logger.log('INFO', `[BigQuery] ✅ Retrieved prices for ${Object.keys(priceMap).length} instruments across ${tickerArray.length} ticker(s)`);
+        }
+        
+        return priceMap;
+    } catch (error) {
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Ticker prices query failed: ${error.message}`);
+        }
+        return null;
+    }
+}
+
 /**
  * Query ticker mappings from BigQuery
  * Returns data in format: { instrumentId: "TICKER", ... }
@@ -2142,6 +2276,8 @@ module.exports = {
     queryHistoryData,
     querySocialData,
     queryAssetPrices,
+    queryAllPricesForDate,
+    queryPricesForTickers,
     queryPIMasterList,
     queryPIRankings,
     queryInstrumentInsights,

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -215,7 +215,10 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
             // Continue - price data is stored, indexer failure is non-critical
         }
         
-        const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
+        const firestoreShardCount = Object.keys(shardUpdates).length;
+        const successMessage = process.env.FIRESTORE_PRICE_FETCH !== 'false'
+            ? `Successfully processed and saved daily prices for ${results.length} instruments to ${firestoreShardCount} Firestore shards.`
+            : `Successfully processed and saved daily prices for ${results.length} instruments (Firestore writes disabled).`;
         logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
         return { success: true, message: successMessage, instrumentsProcessed: results.length };
     } catch (error) { logger.log('ERROR', '[PriceFetcherHelpers] Fatal error during closing price update', { errorMessage: error.message, errorStack: error.stack, headerId: selectedHeader ? selectedHeader.id : 'not-selected' }); throw error;

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -1,10 +1,10 @@
 /**
  * @fileoverview Main pipe: pipe.maintenance.runFetchInsights
- * REFACTORED: Now stateless and receives dependencies.
- * UPDATED: Implements Gzip compression for payloads >900KB.
+ * REFACTORED: Prioritizes BigQuery storage.
+ * UPDATED: Firestore (and compression) is now a fallback only.
  */
 const { FieldValue } = require('@google-cloud/firestore');
-const zlib = require('zlib'); // [NEW] Required for compression
+const zlib = require('zlib'); 
 
 /**
  * Main pipe: pipe.maintenance.runFetchInsights
@@ -25,7 +25,6 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
         try {
             // Extract collection name from registry path: daily_instrument_insights/{date}
             const basePath = getCollectionPath('rootData', 'instrumentInsights', { date: '2025-01-01' });
-            // Path is like "daily_instrument_insights/2025-01-01", extract collection name
             insightsCollectionName = basePath.split('/')[0];
         } catch (e) {
             logger.log('WARN', `[FetchInsightsHelpers] Failed to get collection from registry, using config: ${e.message}`);
@@ -86,54 +85,12 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
         }
         
         const today = new Date().toISOString().slice(0, 10);
-        const docRef = db.collection(insightsCollectionName).doc(today);
-
-        // [FIX] --- COMPRESSION LOGIC START ---
-        
-        // 1. Prepare a standard object to verify size
-        // We use a static date string here so it can be stringified safely
-        const dataObject = {
-            fetchedAt: new Date().toISOString(),
-            instrumentCount: insightsData.length,
-            insights: insightsData
-        };
-
-        const jsonString = JSON.stringify(dataObject);
-        const byteSize = Buffer.byteLength(jsonString);
-        
-        // Firestore limit is ~1MB. We use 900KB as a safety threshold.
-        const SIZE_THRESHOLD = 900 * 1024;
         
-        let firestorePayload;
-
-        if (byteSize > SIZE_THRESHOLD) {
-            logger.log('INFO', `[FetchInsightsHelpers] Payload size ${(byteSize / 1024).toFixed(2)}KB exceeds threshold. Compressing...`);
-            
-            // Gzip the JSON string
-            const compressedBuffer = zlib.gzipSync(Buffer.from(jsonString));
-            
-            // Create the compressed wrapper structure
-            firestorePayload = {
-                fetchedAt: FieldValue.serverTimestamp(), // Keep outer timestamp for indexing
-                instrumentCount: insightsData.length,    // Keep outer metadata for indexing
-                _compressed: true,
-                payload: compressedBuffer
-            };
-            
-            logger.log('INFO', `[FetchInsightsHelpers] Compressed size: ${(compressedBuffer.length / 1024).toFixed(2)}KB.`);
-        } else {
-            // Standard uncompressed write (Legacy behavior)
-            firestorePayload = { 
-                fetchedAt: FieldValue.serverTimestamp(), 
-                instrumentCount: insightsData.length, 
-                insights: insightsData 
-            };
-        }
-        // [FIX] --- COMPRESSION LOGIC END ---
+        // --- STORAGE STRATEGY: BigQuery First, Firestore Fallback ---
+        let storageSuccess = false;
+        let storageMethod = 'NONE';
 
-        await docRef.set(firestorePayload);
-        
-        // Write insights to BigQuery (one row per instrument)
+        // 1. Attempt BigQuery Write
         if (process.env.BIGQUERY_ENABLED !== 'false') {
             try {
                 const { insertRows, ensureInstrumentInsightsTable } = require('../../core/utils/bigquery_utils');
@@ -152,46 +109,87 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
                 const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
                 await insertRows(datasetId, 'instrument_insights', bigqueryRows, logger);
                 
-                logger.log('INFO', `[FetchInsightsHelpers] Successfully stored ${bigqueryRows.length} insight records to BigQuery`);
+                logger.log('INFO', `[FetchInsightsHelpers] ✅ Successfully stored ${bigqueryRows.length} insight records to BigQuery.`);
+                storageSuccess = true;
+                storageMethod = 'BIGQUERY';
             } catch (bqError) {
-                logger.log('WARN', `[FetchInsightsHelpers] BigQuery insights write failed: ${bqError.message}`);
-                // Continue - Firestore write succeeded
+                logger.log('WARN', `[FetchInsightsHelpers] ⚠️ BigQuery write failed. Proceeding to Firestore fallback. Error: ${bqError.message}`);
+                // Proceed to Firestore logic below
             }
         }
-        
-        // Update root data indexer for today's date after insights data is stored
+
+        // 2. Firestore Fallback (If BigQuery failed or is disabled)
+        if (!storageSuccess) {
+            logger.log('INFO', `[FetchInsightsHelpers] 📂 Writing to Firestore (Fallback)...`);
+            const docRef = db.collection(insightsCollectionName).doc(today);
+
+            // [COMPRESSION LOGIC] - Only runs for Firestore
+            const dataObject = {
+                fetchedAt: new Date().toISOString(),
+                instrumentCount: insightsData.length,
+                insights: insightsData
+            };
+
+            const jsonString = JSON.stringify(dataObject);
+            const byteSize = Buffer.byteLength(jsonString);
+            const SIZE_THRESHOLD = 900 * 1024; // 900KB
+            
+            let firestorePayload;
+
+            if (byteSize > SIZE_THRESHOLD) {
+                logger.log('INFO', `[FetchInsightsHelpers] Payload size ${(byteSize / 1024).toFixed(2)}KB exceeds threshold. Compressing for Firestore...`);
+                const compressedBuffer = zlib.gzipSync(Buffer.from(jsonString));
+                firestorePayload = {
+                    fetchedAt: FieldValue.serverTimestamp(),
+                    instrumentCount: insightsData.length,
+                    _compressed: true,
+                    payload: compressedBuffer
+                };
+            } else {
+                firestorePayload = { 
+                    fetchedAt: FieldValue.serverTimestamp(), 
+                    instrumentCount: insightsData.length, 
+                    insights: insightsData 
+                };
+            }
+
+            await docRef.set(firestorePayload);
+            logger.log('INFO', `[FetchInsightsHelpers] ✅ Successfully stored insights to Firestore (Fallback).`);
+            storageSuccess = true;
+            storageMethod = 'FIRESTORE';
+        } else {
+             logger.log('INFO', `[FetchInsightsHelpers] ⏭️ Skipping Firestore write (BigQuery write successful).`);
+        }
+
+        // Update root data indexer
         try {
             const { runRootDataIndexer } = require('../../root-data-indexer/index');
             
             if (!config.rootDataIndexer) {
                 logger.log('WARN', `[FetchInsightsHelpers] Root data indexer config not provided. Skipping index update.`);
             } else {
-                // Merge insights collection name into the full config
                 const indexerConfig = {
                     ...config.rootDataIndexer,
                     collections: {
                         ...config.rootDataIndexer.collections,
-                        insights: insightsCollectionName // Override with actual collection name used
+                        insights: insightsCollectionName
                     },
-                    targetDate: today // Index only today's date for speed
+                    targetDate: today
                 };
                 
-                logger.log('INFO', `[FetchInsightsHelpers] Triggering root data indexer for date ${today} after insights data storage...`);
+                logger.log('INFO', `[FetchInsightsHelpers] Triggering root data indexer for date ${today}...`);
                 const result = await runRootDataIndexer(indexerConfig, dependencies);
                 
                 if (result.success && result.count > 0) {
-                    logger.log('INFO', `[FetchInsightsHelpers] Root data indexer completed successfully for date ${today} (updated ${result.count} dates)`);
-                } else {
-                    logger.log('WARN', `[FetchInsightsHelpers] Root data indexer completed but no dates were updated for ${today}`);
+                    logger.log('INFO', `[FetchInsightsHelpers] Root data indexer completed successfully.`);
                 }
             }
         } catch (indexerError) {
             logger.log('ERROR', `[FetchInsightsHelpers] Failed to run root data indexer for ${today}`, indexerError);
-            // Continue - insights data is stored, indexer failure is non-critical
         }
         
-        const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today}.`; 
-        logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, { documentId: today, instrumentCount: insightsData.length }); 
+        const successMsg = `Successfully fetched and stored ${insightsData.length} instrument insights for ${today} via ${storageMethod}.`; 
+        logger.log('SUCCESS', `[FetchInsightsHelpers] ${successMsg}`, { documentId: today, instrumentCount: insightsData.length, storageMethod }); 
         return { success: true, message: successMsg, instrumentCount: insightsData.length };
         
     } catch (error) {