bulltrackers-module 1.0.710 → 1.0.713

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -96,7 +96,8 @@ async function updateLastUpdated(db, collectionRegistry, cid, userType, dataType
 }
 
 async function processPortfolio(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager, batchManager } = context;
+    const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const url = `${config.ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
     
@@ -132,7 +133,8 @@ async function processPortfolio(context, config, taskData, isPI) {
 }
 
 async function processHistory(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager, batchManager } = context;
+    const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
     const { cid, uuid, today, requestOptions } = taskData;
     const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
     const url = `${config.ETORO_API_HISTORY_URL}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
@@ -145,15 +147,16 @@ async function processHistory(context, config, taskData, isPI) {
         data.PublicHistoryPositions = data.PublicHistoryPositions.filter(p => [0, 1, 5].includes(p.CloseReason));
     }
 
-    if (isPI) await storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data });
-    else await storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data });
+    if (isPI) await storePopularInvestorTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data, bigqueryBatchManager });
+    else await storeSignedInUserTradeHistory({ db, logger, collectionRegistry, cid, date: today, historyData: data, bigqueryBatchManager });
 
     await updateLastUpdated(db, collectionRegistry, cid, isPI ? 'popularInvestor' : 'signedInUser', 'tradeHistory', logger);
     return true;
 }
 
 async function processSocial(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager, batchManager } = context;
+    const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
     
@@ -172,8 +175,11 @@ async function processSocial(context, config, taskData, isPI) {
         }));
 
         // Store posts even if empty (to mark that social fetch was attempted)
-        if (isPI) await storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts });
-        else await storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts });
+        const { batchManager } = context;
+        const bigqueryBatchManager = batchManager?.getBigQueryBatchManager?.() || null;
+        
+        if (isPI) await storePopularInvestorSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts, bigqueryBatchManager });
+        else await storeSignedInUserSocialPosts({ db, logger, collectionRegistry, cid, date: today, posts, bigqueryBatchManager });
         
         // Update lastUpdated timestamp to indicate social fetch completed (even if 0 posts)
         await updateLastUpdated(db, collectionRegistry, cid, isPI ? 'popularInvestor' : 'signedInUser', 'socialPosts', logger);

package/functions/task-engine/utils/bigquery_batch_manager.js

@@ -0,0 +1,201 @@
+/**
+ * @fileoverview BigQuery Batch Manager for Task Engine
+ * 
+ * Batches BigQuery writes and flushes them using LOAD JOBS (FREE).
+ * Integrates with FirestoreBatchManager to flush together.
+ * 
+ * Lessons learned from backfill script:
+ * - Use createLoadJob() not table.load()
+ * - Use temp files for load jobs
+ * - Wait for job completion with getMetadata() polling
+ * - Handle errors gracefully
+ * - Clean up temp files properly
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { 
+    getOrCreateDataset,
+    ensurePortfolioSnapshotsTable,
+    ensureTradeHistorySnapshotsTable,
+    ensureSocialPostSnapshotsTable
+} = require('../../core/utils/bigquery_utils');
+
+class BigQueryBatchManager {
+    constructor(logger) {
+        this.logger = logger;
+        this.datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        
+        // Buffers for each table
+        this.portfolioBuffer = [];
+        this.historyBuffer = [];
+        this.socialBuffer = [];
+        
+        // Track if tables are ensured (avoid repeated checks)
+        this.tablesEnsured = {
+            portfolio: false,
+            history: false,
+            social: false
+        };
+    }
+    
+    /**
+     * Add portfolio row to buffer
+     */
+    async addPortfolioRow(row) {
+        if (!this.tablesEnsured.portfolio) {
+            await ensurePortfolioSnapshotsTable(this.logger);
+            this.tablesEnsured.portfolio = true;
+        }
+        this.portfolioBuffer.push(row);
+    }
+    
+    /**
+     * Add trade history row to buffer
+     */
+    async addHistoryRow(row) {
+        if (!this.tablesEnsured.history) {
+            await ensureTradeHistorySnapshotsTable(this.logger);
+            this.tablesEnsured.history = true;
+        }
+        this.historyBuffer.push(row);
+    }
+    
+    /**
+     * Add social post row to buffer
+     */
+    async addSocialRow(row) {
+        if (!this.tablesEnsured.social) {
+            await ensureSocialPostSnapshotsTable(this.logger);
+            this.tablesEnsured.social = true;
+        }
+        this.socialBuffer.push(row);
+    }
+    
+    /**
+     * Flush a buffer to BigQuery using load job
+     * Uses lessons from backfill: createLoadJob, temp files, proper polling
+     */
+    async _flushBuffer(buffer, tableId, tableName) {
+        if (buffer.length === 0) return 0;
+        
+        const rows = [...buffer]; // Copy buffer
+        buffer.length = 0; // Clear buffer
+        
+        try {
+            const dataset = await getOrCreateDataset(this.datasetId, this.logger);
+            const table = dataset.table(tableId);
+            
+            // Write to temporary file (load jobs require a file, not in-memory data)
+            const tempFile = path.join(os.tmpdir(), `bigquery_${tableId}_${Date.now()}_${Math.random().toString(36).substring(7)}.ndjson`);
+            const ndjson = rows.map(r => JSON.stringify(r)).join('\n');
+            
+            try {
+                fs.writeFileSync(tempFile, ndjson, 'utf8');
+                
+                // Create load job (FREE) - using createLoadJob as learned from backfill
+                const [job] = await table.createLoadJob(tempFile, {
+                    sourceFormat: 'NEWLINE_DELIMITED_JSON',
+                    writeDisposition: 'WRITE_APPEND',
+                    autodetect: false // Use existing table schema
+                });
+                
+                // Wait for job to complete using polling (as learned from backfill)
+                let jobMetadata;
+                const maxAttempts = 60; // 5 minutes max (5 second intervals)
+                const pollInterval = 5000; // 5 seconds
+                
+                for (let attempt = 0; attempt < maxAttempts; attempt++) {
+                    [jobMetadata] = await job.getMetadata();
+                    const state = jobMetadata.status?.state;
+                    
+                    if (state === 'DONE') {
+                        break;
+                    }
+                    
+                    if (state === 'PENDING' || state === 'RUNNING') {
+                        // Wait before next poll
+                        await new Promise(resolve => setTimeout(resolve, pollInterval));
+                    } else {
+                        throw new Error(`Unexpected job state: ${state}`);
+                    }
+                }
+                
+                // Check if we timed out
+                if (jobMetadata.status?.state !== 'DONE') {
+                    throw new Error(`Load job did not complete within ${maxAttempts * pollInterval / 1000} seconds`);
+                }
+                
+                // Check for errors
+                if (jobMetadata.status?.errorResult) {
+                    throw new Error(`Load job failed: ${jobMetadata.status.errorResult.message}`);
+                }
+                
+                const rowsLoaded = jobMetadata.statistics?.load?.outputRows || rows.length;
+                
+                if (this.logger) {
+                    this.logger.log('INFO', `[BigQueryBatch] ✅ Flushed ${rowsLoaded} ${tableName} rows to BigQuery using LOAD JOB (free)`);
+                }
+                
+                return rowsLoaded;
+            } finally {
+                // Clean up temp file (as learned from backfill)
+                try {
+                    if (fs.existsSync(tempFile)) {
+                        fs.unlinkSync(tempFile);
+                    }
+                } catch (cleanupError) {
+                    if (this.logger) {
+                        this.logger.log('WARN', `[BigQueryBatch] Failed to delete temp file ${tempFile}: ${cleanupError.message}`);
+                    }
+                }
+            }
+        } catch (error) {
+            // Log error but don't throw - allow Firestore writes to continue
+            if (this.logger) {
+                this.logger.log('WARN', `[BigQueryBatch] Failed to flush ${tableName} to BigQuery: ${error.message}`);
+            }
+            // Put rows back in buffer for retry on next flush
+            buffer.push(...rows);
+            return 0;
+        }
+    }
+    
+    /**
+     * Flush all buffers to BigQuery
+     * Called by FirestoreBatchManager.flushBatches()
+     */
+    async flushBatches() {
+        if (process.env.BIGQUERY_ENABLED === 'false') {
+            return; // Skip if BigQuery disabled
+        }
+        
+        const results = await Promise.allSettled([
+            this._flushBuffer(this.portfolioBuffer, 'portfolio_snapshots', 'portfolio'),
+            this._flushBuffer(this.historyBuffer, 'trade_history_snapshots', 'history'),
+            this._flushBuffer(this.socialBuffer, 'social_post_snapshots', 'social')
+        ]);
+        
+        const totalFlushed = results
+            .filter(r => r.status === 'fulfilled')
+            .reduce((sum, r) => sum + r.value, 0);
+        
+        if (totalFlushed > 0 && this.logger) {
+            this.logger.log('INFO', `[BigQueryBatch] Flushed ${totalFlushed} total rows to BigQuery`);
+        }
+    }
+    
+    /**
+     * Get buffer sizes (for monitoring)
+     */
+    getBufferSizes() {
+        return {
+            portfolio: this.portfolioBuffer.length,
+            history: this.historyBuffer.length,
+            social: this.socialBuffer.length
+        };
+    }
+}
+
+module.exports = { BigQueryBatchManager };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -5,13 +5,20 @@
  */ 
 
 const { FieldValue } = require('@google-cloud/firestore');
+const { BigQueryBatchManager } = require('./bigquery_batch_manager');
 
 class FirestoreBatchManager { 
     constructor(db, headerManager, logger, config) { 
         this.db            = db; 
         this.headerManager = headerManager; 
         this.logger        = logger; 
-        this.config        = config; 
+        this.config        = config;
+        
+        // Initialize BigQuery batch manager (if enabled)
+        this.bigqueryBatchManager = null;
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            this.bigqueryBatchManager = new BigQueryBatchManager(logger);
+        } 
         
         // State containers for batching
         this.portfolioBatch              = {}; 
@@ -316,9 +323,22 @@ class FirestoreBatchManager {
         }
             
         if (batchOps) await firestoreBatch.commit();
+        
+        // Flush BigQuery batches (if enabled) - runs in parallel with Firestore
+        if (this.bigqueryBatchManager) {
+            await this.bigqueryBatchManager.flushBatches();
+        }
+        
         await this.headerManager.flushPerformanceUpdates();
         this.logger.log('INFO', '[BATCH] All batches flushed successfully.');
     }
+    
+    /**
+     * Get BigQuery batch manager (for adding rows from data_storage_helpers)
+     */
+    getBigQueryBatchManager() {
+        return this.bigqueryBatchManager;
+    }
 }
 
 module.exports = { FirestoreBatchManager };

package/index.js

@@ -61,6 +61,10 @@ const { runBackfillAssetPrices }                          = require('./functions
 const { runRootDataIndexer }                              = require('./functions/root-data-indexer/index');
 // [NEW] Popular Investor Fetcher
 const { runPopularInvestorFetch }                         = require('./functions/fetch-popular-investors/index');
+// [NEW] Backfill Task Engine Data
+const { backfillTaskEngineData }                          = require('./functions/maintenance/backfill-task-engine-data/index');
+const { backfillPIMasterListRankings }                    = require('./functions/maintenance/backfill-pi-master-list-rankings/index');
+const { backfillInstrumentInsights }                      = require('./functions/maintenance/backfill-instrument-insights/index');
 
 // Alert System
 const { handleAlertTrigger, handleComputationResultWrite, checkAndSendAllClearNotifications } = require('./functions/alert-system/index');
@@ -127,7 +131,11 @@ const maintenance = {
   runBackfillAssetPrices,
   runRootDataIndexer,
   // [NEW] Added to maintenance pipe
-  runPopularInvestorFetch
+  runPopularInvestorFetch,
+  // [NEW] BigQuery backfills
+  backfillTaskEngineData,
+  backfillPIMasterListRankings,
+  backfillInstrumentInsights
 };
 
 const proxy = { handlePost };
@@ -140,4 +148,28 @@ const alertSystem = {
 
 module.exports = {
   pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy, alertSystem },
-};
+};
+
+// CLI handling: If running as a script with backfill arguments, execute the backfill
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  
+  // Check if this looks like a backfill command (has --startDate or --endDate)
+  const isBackfillCommand = args.some(arg => 
+    arg.startsWith('--startDate=') || 
+    arg.startsWith('--endDate=') || 
+    arg.startsWith('--dataType=')
+  );
+  
+  if (isBackfillCommand) {
+    // Route to backfill function
+    console.log('🚀 Starting backfill from main entry point...\n');
+    backfillTaskEngineData(null, null).catch(error => {
+      console.error('Fatal error:', error);
+      process.exit(1);
+    });
+  } else {
+    // No recognized command, just export (normal module behavior)
+    // This allows the file to still work as a module when imported
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.710",
+  "version": "1.0.713",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -21,7 +21,10 @@
     "functions/price-backfill/",
     "functions/root-data-indexer/",
     "functions/fetch-popular-investors",
-    "functions/alert-system/"
+    "functions/alert-system/",
+    "functions/maintenance/backfill-instrument-insights",
+    "functions/maintenance/backfill-pi-master-list-rankings",
+    "functions/maintenance/backfill-task-engine-data"
   ],
   "keywords": [
     "bulltrackers",
@@ -46,7 +49,8 @@
     "require-all": "^3.0.0",
     "sharedsetup": "latest",
     "zod": "^4.3.5",
-    "@google-cloud/storage": "^7.18.0"
+    "@google-cloud/storage": "^7.18.0",
+    "@google-cloud/bigquery": "^7.3.0"
   },
   "devDependencies": {
     "bulltracker-deployer": "file:../bulltracker-deployer"