bulltrackers-module 1.0.712 → 1.0.714

package/functions/maintenance/backfill-instrument-insights/index.js

@@ -0,0 +1,180 @@
+/**
+ * @fileoverview Backfill Instrument Insights from Firestore to BigQuery
+ * 
+ * This function reads existing insights data from Firestore
+ * and writes it to BigQuery table.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --startDate=2024-01-01 --endDate=2024-12-31
+ *   node index.js (backfills all dates)
+ * 
+ * Features:
+ *   - Backfills insights for date range (or all dates)
+ *   - Uses load jobs (free) for efficient batching
+ *   - Handles compressed Firestore data
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensureInstrumentInsightsTable,
+    insertRows
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill insights from Firestore to BigQuery for a date range
+ */
+async function backfillInsights(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting insights backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensureInstrumentInsightsTable(logger);
+        
+        const insightsCollection = db.collection('daily_instrument_insights');
+        
+        // Get all insights documents
+        logger.log('INFO', '[Backfill] Fetching insights documents from Firestore...');
+        const snapshot = await insightsCollection.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No insights documents found in Firestore');
+            return { success: false, message: 'No insights found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} insights documents`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each document
+        for (const doc of snapshot.docs) {
+            const dateStr = doc.id; // Document ID is the date (YYYY-MM-DD)
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                const data = tryDecompress(doc.data());
+                const insights = data.insights || [];
+                
+                if (insights.length === 0) {
+                    logger.log('WARN', `[Backfill] No insights in document for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Transform to BigQuery rows
+                const fetchedAt = data.fetchedAt 
+                    ? (data.fetchedAt.toDate ? data.fetchedAt.toDate().toISOString() : data.fetchedAt)
+                    : new Date().toISOString();
+                
+                const bigqueryRows = insights.map(insight => {
+                    return {
+                        date: dateStr,
+                        instrument_id: parseInt(insight.instrumentId, 10),
+                        insights_data: insight, // Store full insight object as JSON
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'instrument_insights', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process insights for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Insights backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Insights backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillInstrumentInsights(startDate = null, endDate = null) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting Instrument Insights backfill...');
+    
+    try {
+        const result = await backfillInsights(startDate, endDate, logger);
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return result;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI handling
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    
+    let startDate = null;
+    let endDate = null;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        }
+    });
+    
+    backfillInstrumentInsights(startDate, endDate)
+        .then(result => {
+            console.log('\n✅ Backfill completed successfully!');
+            console.log('Results:', JSON.stringify(result, null, 2));
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('\n❌ Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillInstrumentInsights, backfillInsights };

package/functions/maintenance/backfill-pi-master-list-rankings/index.js

@@ -0,0 +1,293 @@
+/**
+ * @fileoverview Backfill Popular Investor Master List and Rankings from Firestore to BigQuery
+ * 
+ * This function reads existing master list and rankings data from Firestore
+ * and writes it to BigQuery tables.
+ * 
+ * Usage (Local Node.js script):
+ *   node index.js --masterList=true --rankings=true
+ *   node index.js --masterList=true
+ *   node index.js --rankings=true --startDate=2024-01-01 --endDate=2024-12-31
+ * 
+ * Features:
+ *   - Backfills master list (all PIs from Firestore)
+ *   - Backfills rankings for date range (or all dates)
+ *   - Uses load jobs (free) for efficient batching
+ *   - Does NOT delete any Firestore data
+ */
+
+const { Firestore } = require('@google-cloud/firestore');
+const zlib = require('zlib');
+const { 
+    ensurePIMasterListTable,
+    ensurePIRankingsTable,
+    insertRows,
+    insertRowsWithMerge
+} = require('../../core/utils/bigquery_utils');
+
+const db = new Firestore();
+
+// Helper to decompress Firestore data
+function tryDecompress(data) {
+    if (!data) return null;
+    if (data._compressed && data.payload) {
+        try {
+            const buffer = Buffer.from(data.payload);
+            return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+        } catch (e) {
+            console.error(`[Backfill] Decompression failed: ${e.message}`);
+            return data;
+        }
+    }
+    return data;
+}
+
+/**
+ * Backfill master list from Firestore to BigQuery
+ */
+async function backfillMasterList(logger = console) {
+    logger.log('INFO', '[Backfill] Starting master list backfill...');
+    
+    try {
+        await ensurePIMasterListTable(logger);
+        
+        // Get master list from Firestore
+        const masterListPath = 'system_state/popular_investor_master_list';
+        const masterListRef = db.doc(masterListPath);
+        const masterListDoc = await masterListRef.get();
+        
+        if (!masterListDoc.exists) {
+            logger.log('WARN', '[Backfill] Master list document not found in Firestore');
+            return { success: false, message: 'Master list not found' };
+        }
+        
+        const data = tryDecompress(masterListDoc.data());
+        const investors = data.investors || {};
+        
+        if (Object.keys(investors).length === 0) {
+            logger.log('WARN', '[Backfill] Master list is empty');
+            return { success: false, message: 'Master list is empty' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${Object.keys(investors).length} investors in master list`);
+        
+        // Transform to BigQuery rows
+        const now = new Date().toISOString();
+        
+        // Helper to convert Firestore Timestamp to ISO string
+        const convertTimestamp = (ts) => {
+            if (!ts) return now;
+            if (ts instanceof Date) return ts.toISOString();
+            if (ts.toDate && typeof ts.toDate === 'function') return ts.toDate().toISOString();
+            if (typeof ts === 'string') return ts;
+            // Handle Firestore Timestamp object
+            if (ts._seconds !== undefined) {
+                return new Date(ts._seconds * 1000 + (ts._nanoseconds || 0) / 1000000).toISOString();
+            }
+            return now;
+        };
+        
+        const bigqueryRows = Object.entries(investors).map(([cid, investorData]) => {
+            return {
+                cid: parseInt(cid, 10),
+                username: investorData.username || null,
+                first_seen_at: convertTimestamp(investorData.firstSeenAt),
+                last_seen_at: convertTimestamp(investorData.lastSeenAt),
+                last_updated: now
+            };
+        });
+        
+        // Write to BigQuery using MERGE (updates existing, inserts new)
+        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        await insertRowsWithMerge(datasetId, 'pi_master_list', bigqueryRows, ['cid'], logger);
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Successfully backfilled ${bigqueryRows.length} master list records to BigQuery`);
+        
+        return { success: true, count: bigqueryRows.length };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Master list backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Backfill rankings from Firestore to BigQuery for a date range
+ */
+async function backfillRankings(startDate = null, endDate = null, logger = console) {
+    logger.log('INFO', `[Backfill] Starting rankings backfill${startDate && endDate ? ` from ${startDate} to ${endDate}` : ' (all dates)'}...`);
+    
+    try {
+        await ensurePIRankingsTable(logger);
+        
+        const rankingsCollection = db.collection('popular_investor_rankings');
+        
+        // Get all ranking documents
+        let query = rankingsCollection;
+        
+        if (startDate && endDate) {
+            // Query date range (Firestore doesn't support date range queries directly on doc IDs)
+            // We'll fetch all and filter
+            logger.log('INFO', `[Backfill] Fetching rankings documents (will filter by date range)...`);
+        } else {
+            logger.log('INFO', '[Backfill] Fetching all rankings documents...');
+        }
+        
+        const snapshot = await query.get();
+        
+        if (snapshot.empty) {
+            logger.log('WARN', '[Backfill] No rankings documents found in Firestore');
+            return { success: false, message: 'No rankings found' };
+        }
+        
+        logger.log('INFO', `[Backfill] Found ${snapshot.size} rankings documents`);
+        
+        let totalRows = 0;
+        let processedDates = 0;
+        let skippedDates = 0;
+        
+        // Process each document
+        for (const doc of snapshot.docs) {
+            const dateStr = doc.id; // Document ID is the date (YYYY-MM-DD)
+            
+            // Filter by date range if provided
+            if (startDate && endDate) {
+                if (dateStr < startDate || dateStr > endDate) {
+                    skippedDates++;
+                    continue;
+                }
+            }
+            
+            try {
+                const data = tryDecompress(doc.data());
+                const items = data.Items || [];
+                
+                if (items.length === 0) {
+                    logger.log('WARN', `[Backfill] No items in rankings for ${dateStr}`);
+                    skippedDates++;
+                    continue;
+                }
+                
+                // Transform to BigQuery rows
+                // Helper to convert Firestore Timestamp to ISO string
+                const convertTimestamp = (ts) => {
+                    if (!ts) return new Date().toISOString();
+                    if (ts instanceof Date) return ts.toISOString();
+                    if (ts.toDate && typeof ts.toDate === 'function') return ts.toDate().toISOString();
+                    if (typeof ts === 'string') return ts;
+                    return new Date().toISOString();
+                };
+                
+                const fetchedAt = convertTimestamp(data.fetchedAt);
+                
+                const bigqueryRows = items.map((item, index) => {
+                    return {
+                        date: dateStr,
+                        pi_id: parseInt(item.CustomerId, 10),
+                        username: item.UserName || null,
+                        rank: item.Rank || (index + 1), // Use Rank from item, or position in array
+                        category: item.Category || null,
+                        rankings_data: item, // Store full item data as JSON
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                // Write to BigQuery using load jobs (free, batched)
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'pi_rankings', bigqueryRows, logger);
+                
+                totalRows += bigqueryRows.length;
+                processedDates++;
+                
+                if (processedDates % 10 === 0) {
+                    logger.log('INFO', `[Backfill] Processed ${processedDates} dates, ${totalRows} rows so far...`);
+                }
+            } catch (dateError) {
+                logger.log('ERROR', `[Backfill] Failed to process rankings for ${dateStr}: ${dateError.message}`);
+                skippedDates++;
+            }
+        }
+        
+        logger.log('SUCCESS', `[Backfill] ✅ Rankings backfill complete: ${processedDates} dates processed, ${totalRows} rows, ${skippedDates} skipped`);
+        
+        return { success: true, processedDates, totalRows, skippedDates };
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Rankings backfill failed: ${error.message}`);
+        throw error;
+    }
+}
+
+/**
+ * Main entry point
+ */
+async function backfillPIMasterListRankings(startDate = null, endDate = null, masterList = true, rankings = true) {
+    const logger = {
+        log: (level, message, ...args) => {
+            const timestamp = new Date().toISOString();
+            console.log(`[${timestamp}] [${level}] ${message}`, ...args);
+        }
+    };
+    
+    logger.log('INFO', '[Backfill] Starting PI Master List and Rankings backfill...');
+    
+    const results = {
+        masterList: null,
+        rankings: null
+    };
+    
+    try {
+        // Backfill master list
+        if (masterList) {
+            results.masterList = await backfillMasterList(logger);
+        } else {
+            logger.log('INFO', '[Backfill] Skipping master list backfill');
+        }
+        
+        // Backfill rankings
+        if (rankings) {
+            results.rankings = await backfillRankings(startDate, endDate, logger);
+        } else {
+            logger.log('INFO', '[Backfill] Skipping rankings backfill');
+        }
+        
+        logger.log('SUCCESS', '[Backfill] ✅ All backfills completed!');
+        return results;
+    } catch (error) {
+        logger.log('ERROR', `[Backfill] Fatal error: ${error.message}`);
+        throw error;
+    }
+}
+
+// CLI handling
+if (require.main === module) {
+    const args = process.argv.slice(2);
+    
+    let startDate = null;
+    let endDate = null;
+    let masterList = true;
+    let rankings = true;
+    
+    args.forEach(arg => {
+        if (arg.startsWith('--startDate=')) {
+            startDate = arg.split('=')[1];
+        } else if (arg.startsWith('--endDate=')) {
+            endDate = arg.split('=')[1];
+        } else if (arg.startsWith('--masterList=')) {
+            masterList = arg.split('=')[1] === 'true';
+        } else if (arg.startsWith('--rankings=')) {
+            rankings = arg.split('=')[1] === 'true';
+        }
+    });
+    
+    backfillPIMasterListRankings(startDate, endDate, masterList, rankings)
+        .then(results => {
+            console.log('\n✅ Backfill completed successfully!');
+            console.log('Results:', JSON.stringify(results, null, 2));
+            process.exit(0);
+        })
+        .catch(error => {
+            console.error('\n❌ Backfill failed:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = { backfillPIMasterListRankings, backfillMasterList, backfillRankings };

package/functions/maintenance/backfill-task-engine-data/README.md

@@ -0,0 +1,72 @@
+# Backfill Task Engine Data
+
+Backfills portfolio, trade history, and social post data from Firestore to BigQuery.
+
+## Local Usage
+
+Run from the project root directory:
+
+```bash
+# Backfill all data types
+node Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/maintenance/backfill-task-engine-data/index.js \
+  --startDate=2024-01-01 \
+  --endDate=2024-12-31 \
+  --dataType=all
+
+# Backfill only portfolio data
+node Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/maintenance/backfill-task-engine-data/index.js \
+  --startDate=2024-01-01 \
+  --endDate=2024-12-31 \
+  --dataType=portfolio
+
+# Backfill only trade history
+node Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/maintenance/backfill-task-engine-data/index.js \
+  --startDate=2024-01-01 \
+  --endDate=2024-12-31 \
+  --dataType=history
+
+# Backfill only social posts
+node Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/maintenance/backfill-task-engine-data/index.js \
+  --startDate=2024-01-01 \
+  --endDate=2024-12-31 \
+  --dataType=social
+```
+
+## Prerequisites
+
+1. **Authentication**: Make sure you're authenticated with Google Cloud:
+   ```bash
+   gcloud auth application-default login
+   ```
+
+2. **Environment Variables** (optional):
+   ```bash
+   export BIGQUERY_DATASET_ID=bulltrackers_data
+   export GCP_PROJECT_ID=stocks-12345
+   ```
+
+3. **Dependencies**: Make sure you're in the correct directory with node_modules installed.
+
+## Cloud Function Usage
+
+The script also works as a Cloud Function with HTTP trigger:
+
+```
+GET /backfill-task-engine-data?startDate=2024-01-01&endDate=2024-12-31&dataType=all
+```
+
+## Features
+
+- ✅ **Resume capability**: Tracks progress in Firestore, can resume if interrupted
+- ✅ **Batch processing**: Processes dates in batches to avoid timeouts
+- ✅ **Safe**: Does NOT delete any Firestore data
+- ✅ **Progress tracking**: Logs progress and summary at the end
+
+## Progress Tracking
+
+Progress is stored in Firestore at:
+- `backfill_progress/portfolio/dates/{date}`
+- `backfill_progress/history/dates/{date}`
+- `backfill_progress/social/dates/{date}`
+
+If a date is already backfilled, it will be skipped automatically.