bulltrackers-module 1.0.710 → 1.0.713

package/functions/core/utils/firestore_utils.js

@@ -281,8 +281,8 @@ function needsUpdate(lastUpdated, todayStr) {
 
 /**
  * [NEW] Fetches Popular Investors from the master list and filters by last updated times.
- * UPDATED: Uses the master list as single source of truth, then checks last updated timestamps
- * for each data type (portfolio, tradeHistory, socialPosts) against TODAY to determine who needs updating.
+ * UPDATED: Uses BigQuery first for master list, then checks BigQuery for today's data to filter users.
+ * Falls back to Firestore if BigQuery is disabled or fails.
  */
 async function getPopularInvestorsToUpdate(dependencies, config) {
     const { db, logger, collectionRegistry } = dependencies;
@@ -291,53 +291,122 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
     logger.log('INFO', `[Core Utils] Getting Popular Investors to update (Checking against date: ${todayStr})...`);
     
     try {
-        // Get the master list of Popular Investors
-        let masterListPath = 'system_state/popular_investor_master_list';
+        let investors = {};
+        let masterListSource = 'UNKNOWN';
         
-        if (collectionRegistry && collectionRegistry.getCollectionPath) {
+        // =========================================================================
+        // BIGQUERY FIRST: Try BigQuery for master list
+        // =========================================================================
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
             try {
-                masterListPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
-            } catch (err) {
-                logger.log('WARN', `[Core Utils] Failed to get master list path from registry, using default: ${err.message}`);
+                const { queryPIMasterList } = require('../../core/utils/bigquery_utils');
+                const bigqueryMasterList = await queryPIMasterList(logger);
+                
+                if (bigqueryMasterList && Object.keys(bigqueryMasterList).length > 0) {
+                    investors = bigqueryMasterList;
+                    masterListSource = 'BIGQUERY';
+                    logger.log('INFO', `[Core Utils] ✅ Loaded PI master list from BigQuery: ${Object.keys(investors).length} investors`);
+                }
+            } catch (bqError) {
+                logger.log('WARN', `[Core Utils] BigQuery master list query failed, falling back to Firestore: ${bqError.message}`);
             }
         }
         
-        const masterListRef = db.doc(masterListPath);
-        const masterListDoc = await masterListRef.get();
-        
-        if (!masterListDoc.exists) {
-            logger.log('WARN', `[Core Utils] Master list not found. Falling back to legacy method.`);
-            return await getPopularInvestorsToUpdateLegacy(dependencies, config);
+        // =========================================================================
+        // FIRESTORE FALLBACK: If BigQuery didn't return data
+        // =========================================================================
+        if (Object.keys(investors).length === 0) {
+            let masterListPath = 'system_state/popular_investor_master_list';
+            
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    masterListPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+                } catch (err) {
+                    logger.log('WARN', `[Core Utils] Failed to get master list path from registry, using default: ${err.message}`);
+                }
+            }
+            
+            const masterListRef = db.doc(masterListPath);
+            const masterListDoc = await masterListRef.get();
+            
+            if (!masterListDoc.exists) {
+                logger.log('WARN', `[Core Utils] Master list not found in Firestore. Falling back to legacy method.`);
+                return await getPopularInvestorsToUpdateLegacy(dependencies, config);
+            }
+            
+            const masterListData = masterListDoc.data();
+            investors = masterListData.investors || {};
+            masterListSource = 'FIRESTORE';
+            logger.log('INFO', `[Core Utils] Loaded PI master list from Firestore: ${Object.keys(investors).length} investors`);
         }
         
-        const masterListData = masterListDoc.data();
-        const investors = masterListData.investors || {};
-        
         if (Object.keys(investors).length === 0) {
             logger.log('WARN', `[Core Utils] Master list is empty. Returning empty array.`);
             return [];
         }
         
+        // =========================================================================
+        // FILTER: Check BigQuery for today's data to determine who needs updating
+        // =========================================================================
         const targets = [];
         let skippedCount = 0;
         
+        // Get today's data from BigQuery to check who's already updated
+        let todayPortfolioUsers = new Set();
+        let todayHistoryUsers = new Set();
+        let todaySocialUsers = new Set();
+        
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const { queryPortfolioData, queryHistoryData, querySocialData } = require('../../core/utils/bigquery_utils');
+                
+                // Query BigQuery for today's data (only for POPULAR_INVESTOR user type)
+                const [portfolioData, historyData, socialData] = await Promise.all([
+                    queryPortfolioData(todayStr, null, ['POPULAR_INVESTOR'], logger).catch(() => null),
+                    queryHistoryData(todayStr, null, ['POPULAR_INVESTOR'], logger).catch(() => null),
+                    querySocialData(todayStr, null, ['POPULAR_INVESTOR'], logger).catch(() => null)
+                ]);
+                
+                if (portfolioData) {
+                    todayPortfolioUsers = new Set(Object.keys(portfolioData));
+                }
+                if (historyData) {
+                    todayHistoryUsers = new Set(Object.keys(historyData));
+                }
+                if (socialData) {
+                    todaySocialUsers = new Set(Object.keys(socialData));
+                }
+                
+                logger.log('INFO', `[Core Utils] BigQuery filter: ${todayPortfolioUsers.size} portfolios, ${todayHistoryUsers.size} histories, ${todaySocialUsers.size} social posts for ${todayStr}`);
+            } catch (bqError) {
+                logger.log('WARN', `[Core Utils] BigQuery filter query failed, using Firestore fallback: ${bqError.message}`);
+            }
+        }
+        
+        // Process each investor from master list
         for (const [cid, piData] of Object.entries(investors)) {
             const username = piData.username || String(cid);
             
-            // Construct path manually to avoid registry lookup errors on "profile"
-            const piDocPath = `PopularInvestors/${cid}`;
-            const piDoc = await db.doc(piDocPath).get();
-            
-            // Default: All need update
-            let components = { portfolio: true, tradeHistory: true, socialPosts: true };
+            // Check if already updated today (BigQuery first, then Firestore)
+            let components = {
+                portfolio: !todayPortfolioUsers.has(cid),
+                tradeHistory: !todayHistoryUsers.has(cid),
+                socialPosts: !todaySocialUsers.has(cid)
+            };
             
-            if (piDoc.exists) {
-                const data = piDoc.data();
-                const lastUpdated = data.lastUpdated || {};
+            // If BigQuery didn't have data, check Firestore
+            if (todayPortfolioUsers.size === 0 && todayHistoryUsers.size === 0 && todaySocialUsers.size === 0) {
+                const piDocPath = `PopularInvestors/${cid}`;
+                const piDoc = await db.doc(piDocPath).get();
                 
-                components.portfolio = needsUpdate(lastUpdated.portfolio, todayStr);
-                components.tradeHistory = needsUpdate(lastUpdated.tradeHistory, todayStr);
-                components.socialPosts = needsUpdate(lastUpdated.socialPosts, todayStr);
+                if (piDoc.exists) {
+                    const data = piDoc.data();
+                    const lastUpdated = data.lastUpdated || {};
+                    
+                    components.portfolio = needsUpdate(lastUpdated.portfolio, todayStr);
+                    components.tradeHistory = needsUpdate(lastUpdated.tradeHistory, todayStr);
+                    components.socialPosts = needsUpdate(lastUpdated.socialPosts, todayStr);
+                }
             }
             
             // Only add if at least one component needs updating
@@ -348,7 +417,7 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
             }
         }
         
-        logger.log('INFO', `[Core Utils] Found ${Object.keys(investors).length} Popular Investors in master list. Skipped ${skippedCount} (updated today). ${targets.length} queued for update.`);
+        logger.log('INFO', `[Core Utils] Found ${Object.keys(investors).length} Popular Investors in master list (${masterListSource}). Skipped ${skippedCount} (updated today). ${targets.length} queued for update.`);
         return targets;
         
     } catch (error) {

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -16,11 +16,24 @@ const SHARD_SIZE = 40;
  * @returns {Promise<{success: boolean, message: string, instrumentsProcessed?: number}>}
  */
 exports.fetchAndStorePrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager, collectionRegistry } = dependencies;
+    const { db, logger, headerManager, proxyManager, collectionRegistry, calculationUtils } = dependencies;
     logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
     let selectedHeader = null;
     let wasSuccessful = false;
     
+    // Load instrument mappings for ticker information (needed for BigQuery)
+    let instrumentMappings = null;
+    if (process.env.BIGQUERY_ENABLED !== 'false' && calculationUtils?.loadInstrumentMappings) {
+        try {
+            instrumentMappings = await calculationUtils.loadInstrumentMappings();
+            if (instrumentMappings?.instrumentToTicker) {
+                logger.log('INFO', `[PriceFetcherHelpers] Loaded ${Object.keys(instrumentMappings.instrumentToTicker).length} instrument mappings for ticker lookup`);
+            }
+        } catch (mappingError) {
+            logger.log('WARN', `[PriceFetcherHelpers] Failed to load instrument mappings: ${mappingError.message}. Ticker will be set to 'unknown_{instrumentId}'`);
+        }
+    }
+    
     // Get collection names from registry if available, fallback to hardcoded
     const { getCollectionPath } = collectionRegistry || {};
     let priceCollectionName = 'asset_prices';
@@ -50,23 +63,82 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         wasSuccessful = true;
         const results = await response.json();
         if (!Array.isArray(results)) { throw new Error('Invalid response format from API. Expected an array.'); }        
-        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Sharding...`);
-        const shardUpdates = {}; 
+        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Processing for BigQuery...`);
+        
+        // Transform daily prices to BigQuery rows
+        const fetchedAt = new Date().toISOString();
+        const bigqueryRows = [];
+        const shardUpdates = {}; // Keep for Firestore backward compatibility
+        
         for (const instrumentData of results) {
             const dailyData = instrumentData?.ClosingPrices?.Daily;
             const instrumentId = instrumentData.InstrumentId;
+            
             if (instrumentId && dailyData?.Price && dailyData?.Date) {
                 const instrumentIdStr = String(instrumentId);
-                const dateKey = dailyData.Date.substring(0, 10);
-                const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
-                if (!shardUpdates[shardId]) { shardUpdates[shardId] = {}; }
-                const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
-                const updatePath = `${instrumentIdStr}.lastUpdated`;
-                shardUpdates[shardId][pricePath] = dailyData.Price;
-                shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp(); } }
-        const batchPromises = [];
-        for (const shardId in shardUpdates) { const docRef = db.collection(priceCollectionName).doc(shardId); const payload = shardUpdates[shardId]; batchPromises.push(docRef.update(payload)); }
-        await Promise.all(batchPromises);
+                const dateKey = dailyData.Date.substring(0, 10); // Extract YYYY-MM-DD
+                
+                // Get ticker from mappings if available
+                let ticker = `unknown_${instrumentId}`;
+                if (instrumentMappings?.instrumentToTicker?.[instrumentIdStr]) {
+                    ticker = instrumentMappings.instrumentToTicker[instrumentIdStr];
+                } else if (instrumentData.Ticker) {
+                    ticker = instrumentData.Ticker;
+                }
+                
+                // Prepare BigQuery row
+                bigqueryRows.push({
+                    date: dateKey,
+                    instrument_id: parseInt(instrumentId, 10),
+                    ticker: ticker,
+                    price: dailyData.Price,
+                    open: null, // Daily API doesn't provide OHLC, only closing price
+                    high: null,
+                    low: null,
+                    close: dailyData.Price,
+                    volume: null,
+                    fetched_at: fetchedAt
+                });
+                
+                // Also prepare Firestore update for backward compatibility
+                if (process.env.FIRESTORE_PRICE_FETCH !== 'false') {
+                    const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
+                    if (!shardUpdates[shardId]) { shardUpdates[shardId] = {}; }
+                    const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
+                    const updatePath = `${instrumentIdStr}.lastUpdated`;
+                    shardUpdates[shardId][pricePath] = dailyData.Price;
+                    shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp();
+                }
+            }
+        }
+        
+        // Write to BigQuery using load jobs (free, batched)
+        if (process.env.BIGQUERY_ENABLED !== 'false' && bigqueryRows.length > 0) {
+            try {
+                const { insertRows, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
+                await ensureAssetPricesTable(logger);
+                
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                
+                logger.log('INFO', `[PriceFetcherHelpers] Successfully stored ${bigqueryRows.length} daily price records to BigQuery`);
+            } catch (bqError) {
+                logger.log('ERROR', `[PriceFetcherHelpers] BigQuery write failed: ${bqError.message}`);
+                // Continue - don't fail the entire fetch for BigQuery errors
+            }
+        }
+        
+        // Also write to Firestore for backward compatibility (if needed)
+        if (process.env.FIRESTORE_PRICE_FETCH !== 'false' && Object.keys(shardUpdates).length > 0) {
+            const batchPromises = [];
+            for (const shardId in shardUpdates) { 
+                const docRef = db.collection(priceCollectionName).doc(shardId); 
+                const payload = shardUpdates[shardId]; 
+                batchPromises.push(docRef.update(payload)); 
+            }
+            await Promise.all(batchPromises);
+            logger.log('INFO', `[PriceFetcherHelpers] Also stored prices to ${batchPromises.length} Firestore shards`);
+        }
         
         // Extract all dates from the price data and create a date tracking document
         const priceDatesSet = new Set();

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -133,6 +133,32 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
 
         await docRef.set(firestorePayload);
         
+        // Write insights to BigQuery (one row per instrument)
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const { insertRows, ensureInstrumentInsightsTable } = require('../../core/utils/bigquery_utils');
+                await ensureInstrumentInsightsTable(logger);
+                
+                const fetchedAt = new Date().toISOString();
+                const bigqueryRows = insightsData.map(insight => {
+                    return {
+                        date: today,
+                        instrument_id: parseInt(insight.instrumentId, 10),
+                        insights_data: insight, // Store full insight object as JSON
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'instrument_insights', bigqueryRows, logger);
+                
+                logger.log('INFO', `[FetchInsightsHelpers] Successfully stored ${bigqueryRows.length} insight records to BigQuery`);
+            } catch (bqError) {
+                logger.log('WARN', `[FetchInsightsHelpers] BigQuery insights write failed: ${bqError.message}`);
+                // Continue - Firestore write succeeded
+            }
+        }
+        
         // Update root data indexer for today's date after insights data is stored
         try {
             const { runRootDataIndexer } = require('../../root-data-indexer/index');

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -315,6 +315,35 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
 
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${finalRankingsCollectionName}/${today}${firestorePayload._compressed ? ' (compressed)' : ''}`);
             
+            // Write rankings to BigQuery (one row per PI)
+            if (process.env.BIGQUERY_ENABLED !== 'false') {
+                try {
+                    const { insertRows, ensurePIRankingsTable } = require('../../core/utils/bigquery_utils');
+                    await ensurePIRankingsTable(logger);
+                    
+                    const fetchedAt = new Date().toISOString();
+                    const bigqueryRows = data.Items.map((item, index) => {
+                        return {
+                            date: today,
+                            pi_id: parseInt(item.CustomerId, 10),
+                            username: item.UserName || null,
+                            rank: index + 1, // Rank is position in array (1-indexed)
+                            category: item.Category || null,
+                            rankings_data: item, // Store full item data as JSON
+                            fetched_at: fetchedAt
+                        };
+                    });
+                    
+                    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                    await insertRows(datasetId, 'pi_rankings', bigqueryRows, logger);
+                    
+                    logger.log('INFO', `[PopularInvestorFetch] Successfully stored ${bigqueryRows.length} ranking records to BigQuery`);
+                } catch (bqError) {
+                    logger.log('WARN', `[PopularInvestorFetch] BigQuery rankings write failed: ${bqError.message}`);
+                    // Continue - Firestore write succeeded
+                }
+            }
+            
             // Update the master list of Popular Investors
             // Use batched writes to avoid 500 field transform limit
             try {
@@ -426,6 +455,43 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
                 }
                 
                 logger.log('SUCCESS', `[PopularInvestorFetch] Updated master list: ${newInvestorsCount} new, ${updatedInvestorsCount} updated. Total unique PIs: ${Object.keys({ ...existingInvestors, ...investorsToUpdate }).length}`);
+                
+                // Write master list updates to BigQuery
+                if (process.env.BIGQUERY_ENABLED !== 'false' && Object.keys(investorsToUpdate).length > 0) {
+                    try {
+                        const { insertRowsWithMerge, ensurePIMasterListTable } = require('../../core/utils/bigquery_utils');
+                        await ensurePIMasterListTable(logger);
+                        
+                        const now = new Date().toISOString();
+                        const bigqueryRows = Object.entries(investorsToUpdate).map(([cid, investorData]) => {
+                            // Handle Firestore Timestamp objects
+                            const convertTimestamp = (ts) => {
+                                if (!ts) return now;
+                                if (ts instanceof Date) return ts.toISOString();
+                                if (ts.toDate && typeof ts.toDate === 'function') return ts.toDate().toISOString();
+                                if (typeof ts === 'string') return ts;
+                                return now;
+                            };
+                            
+                            return {
+                                cid: parseInt(cid, 10),
+                                username: investorData.username,
+                                first_seen_at: convertTimestamp(investorData.firstSeenAt),
+                                last_seen_at: convertTimestamp(investorData.lastSeenAt),
+                                last_updated: now
+                            };
+                        });
+                        
+                        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                        // Use MERGE to update existing records or insert new ones
+                        await insertRowsWithMerge(datasetId, 'pi_master_list', bigqueryRows, ['cid'], logger);
+                        
+                        logger.log('INFO', `[PopularInvestorFetch] Successfully stored ${bigqueryRows.length} master list records to BigQuery`);
+                    } catch (bqError) {
+                        logger.log('WARN', `[PopularInvestorFetch] BigQuery master list write failed: ${bqError.message}`);
+                        // Continue - Firestore write succeeded
+                    }
+                }
             } catch (masterListError) {
                 logger.log('WARN', `[PopularInvestorFetch] Failed to update master list: ${masterListError.message}`);
                 // Non-critical, continue

package/functions/maintenance/backfill-instrument-insights/index.js

@@ -0,0 +1,180 @@
+/**
+ * @fileoverview Backfill Instrument Insights from Firestore to BigQuery
+ * 
+ * This function reads existing insights data from Firestore
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ *   node index.js (backfills all dates)
+ * 
+ * Features:
+ *   - Backfills insights for date range (or all dates)
+ *   - Uses load jobs (free) for efficient batching
+ *   - Handles compressed Firestore data
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensureInstrumentInsightsTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill insights from Firestore to BigQuery for a date range
+ */
+async function backfillInsights(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting insights backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensureInstrumentInsightsTable(logger);
+        
+        const insightsCollection = db.collection('daily_instrument_insights');
+        
+        // Get all insights documents
+        logger.log('INFO', '[Backfill] Fetching insights documents from Firestore...');
+        const snapshot = await insightsCollection.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No insights documents found in Firestore');
+            return { success: false, message: 'No insights found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} insights documents`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each document
+        for (const doc of snapshot.docs) {
+            const dateStr = doc.id; // Document ID is the date (YYYY-MM-DD)
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                const data = tryDecompress(doc.data());
+                const insights = data.insights || [];
+                
+                if (insights.length === 0) {
+                    logger.log('WARN', `[Backfill] No insights in document for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Transform to BigQuery rows
+                const fetchedAt = data.fetchedAt 
+                    ? (data.fetchedAt.toDate ? data.fetchedAt.toDate().toISOString() : data.fetchedAt)
+                    : new Date().toISOString();
+                
+                const bigqueryRows = insights.map(insight => {
+                    return {
+                        date: dateStr,
+                        instrument_id: parseInt(insight.instrumentId, 10),
+                        insights_data: insight, // Store full insight object as JSON
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'instrument_insights', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process insights for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Insights backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Insights backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillInstrumentInsights(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting Instrument Insights backfill...');
+    
+    try {
+        const result = await backfillInsights(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI handling
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillInstrumentInsights(startDate, endDate)
+        .then(result => {
+            console.log('\n✅ Backfill completed successfully!');
+            console.log('Results:', JSON.stringify(result, null, 2));
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('\n❌ Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillInstrumentInsights, backfillInsights };