bulltrackers-module 1.0.710 → 1.0.712

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -5,13 +5,20 @@
  */ 
 
 const { FieldValue } = require('@google-cloud/firestore');
+const { BigQueryBatchManager } = require('./bigquery_batch_manager');
 
 class FirestoreBatchManager { 
     constructor(db, headerManager, logger, config) { 
         this.db            = db; 
         this.headerManager = headerManager; 
         this.logger        = logger; 
-        this.config        = config; 
+        this.config        = config;
+        
+        // Initialize BigQuery batch manager (if enabled)
+        this.bigqueryBatchManager = null;
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            this.bigqueryBatchManager = new BigQueryBatchManager(logger);
+        } 
         
         // State containers for batching
         this.portfolioBatch              = {}; 
@@ -316,9 +323,22 @@ class FirestoreBatchManager {
         }
             
         if (batchOps) await firestoreBatch.commit();
+        
+        // Flush BigQuery batches (if enabled) - runs in parallel with Firestore
+        if (this.bigqueryBatchManager) {
+            await this.bigqueryBatchManager.flushBatches();
+        }
+        
         await this.headerManager.flushPerformanceUpdates();
         this.logger.log('INFO', '[BATCH] All batches flushed successfully.');
     }
+    
+    /**
+     * Get BigQuery batch manager (for adding rows from data_storage_helpers)
+     */
+    getBigQueryBatchManager() {
+        return this.bigqueryBatchManager;
+    }
 }
 
 module.exports = { FirestoreBatchManager };

package/index.js

@@ -61,6 +61,10 @@ const { runBackfillAssetPrices }                          = require('./functions
 const { runRootDataIndexer }                              = require('./functions/root-data-indexer/index');
 // [NEW] Popular Investor Fetcher
 const { runPopularInvestorFetch }                         = require('./functions/fetch-popular-investors/index');
+// [NEW] Backfill Task Engine Data
+const { backfillTaskEngineData }                          = require('./functions/maintenance/backfill-task-engine-data/index');
+const { backfillPIMasterListRankings }                    = require('./functions/maintenance/backfill-pi-master-list-rankings/index');
+const { backfillInstrumentInsights }                      = require('./functions/maintenance/backfill-instrument-insights/index');
 
 // Alert System
 const { handleAlertTrigger, handleComputationResultWrite, checkAndSendAllClearNotifications } = require('./functions/alert-system/index');
@@ -127,7 +131,11 @@ const maintenance = {
   runBackfillAssetPrices,
   runRootDataIndexer,
   // [NEW] Added to maintenance pipe
-  runPopularInvestorFetch
+  runPopularInvestorFetch,
+  // [NEW] BigQuery backfills
+  backfillTaskEngineData,
+  backfillPIMasterListRankings,
+  backfillInstrumentInsights
 };
 
 const proxy = { handlePost };
@@ -140,4 +148,28 @@ const alertSystem = {
 
 module.exports = {
   pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy, alertSystem },
-};
+};
+
+// CLI handling: If running as a script with backfill arguments, execute the backfill
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  
+  // Check if this looks like a backfill command (has --startDate or --endDate)
+  const isBackfillCommand = args.some(arg => 
+    arg.startsWith('--startDate=') || 
+    arg.startsWith('--endDate=') || 
+    arg.startsWith('--dataType=')
+  );
+  
+  if (isBackfillCommand) {
+    // Route to backfill function
+    console.log('🚀 Starting backfill from main entry point...\n');
+    backfillTaskEngineData(null, null).catch(error => {
+      console.error('Fatal error:', error);
+      process.exit(1);
+    });
+  } else {
+    // No recognized command, just export (normal module behavior)
+    // This allows the file to still work as a module when imported
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.710",
+  "version": "1.0.712",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -46,7 +46,8 @@
     "require-all": "^3.0.0",
     "sharedsetup": "latest",
     "zod": "^4.3.5",
-    "@google-cloud/storage": "^7.18.0"
+    "@google-cloud/storage": "^7.18.0",
+    "@google-cloud/bigquery": "^7.3.0"
   },
   "devDependencies": {
     "bulltracker-deployer": "file:../bulltracker-deployer"