bulltrackers-module 1.0.475 → 1.0.476

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,6 +1,7 @@
-# Data Feeder Pipeline (V3.1 - Syntax Fixed)
+# Data Feeder Pipeline (V3.2 - Automatic Root Data Indexing)
 # Starts at 22:00 UTC via Cloud Scheduler.
-# Fixes: Split assign/call steps and corrected assign syntax.
+# UPDATED: Removed intermediate root data indexer calls - each fetcher now automatically triggers indexing.
+# Only the global verification run at midnight remains.
 
 main:
   params: [input]
@@ -45,17 +46,7 @@ main:
     - wait_10_after_price:
         call: sys.sleep
         args: { seconds: 600 } # 10 Minutes
-
-    # FIX 1: Split assign and call
-    - prepare_index_price:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_price:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Price fetcher now automatically triggers root data indexer after completion
 
     - wait_10_before_insights:
         call: sys.sleep
@@ -78,17 +69,7 @@ main:
     - wait_10_after_insights:
         call: sys.sleep
         args: { seconds: 600 }
-
-    # FIX 2: Split assign and call
-    - prepare_index_insights:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_insights:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Insights fetcher now automatically triggers root data indexer after completion
 
     # ==========================================
     # PHASE 2: WAIT FOR MIDNIGHT
@@ -124,17 +105,7 @@ main:
     - wait_10_after_rankings:
         call: sys.sleep
         args: { seconds: 600 }
-
-    # FIX 3: Split assign and call
-    - prepare_index_rankings:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_rankings:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Popular investor rankings fetcher now automatically triggers root data indexer after completion
 
     - phase_0000_social:
         try:
@@ -158,7 +129,7 @@ main:
         call: http.post
         args:
           url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          # No targetDate = Global Run
+          # No targetDate = Global verification run (all intermediate indexing is now automatic)
           auth: { type: OIDC }
 
     # ==========================================
@@ -194,17 +165,7 @@ main:
               - wait_10_in_loop:
                   call: sys.sleep
                   args: { seconds: 600 }
-              
-              # FIX 4: Split assign and call
-              - prepare_index_loop:
-                   assign:
-                     - today: '${text.split(time.format(sys.now()), "T")[0]}'
-              - index_today_in_loop:
-                   call: http.post
-                   args:
-                     url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-                     body: { targetDate: '${today}' }
-                     auth: { type: OIDC }
+                  # NOTE: Social tasks are handled by task engine, which automatically triggers root data indexer after batch completion
               
               # FIX 5: Correct assign syntax (must be a list)
               - increment_loop:

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -113,6 +113,37 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
             });
 
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${rankingsCollectionName}/${today}`);
+            
+            // Update root data indexer for today's date after rankings data is stored
+            try {
+                const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                // Access rootDataIndexer from config (passed from index.js) or use defaults
+                // Using bracket notation to avoid TypeScript errors
+                let rootDataIndexerConfig;
+                if (config && typeof config === 'object' && config['rootDataIndexer']) {
+                    rootDataIndexerConfig = config['rootDataIndexer'];
+                } else {
+                    rootDataIndexerConfig = {
+                        availabilityCollection: 'system_root_data_index',
+                        earliestDate: '2025-08-01',
+                        collections: {
+                            piRankings: rankingsCollectionName
+                        }
+                    };
+                }
+                
+                const indexerConfig = Object.assign({}, rootDataIndexerConfig, {
+                    targetDate: today // Index only today's date for speed
+                });
+                
+                logger.log('INFO', `[PopularInvestorFetch] Triggering root data indexer for date ${today} after rankings data storage...`);
+                await runRootDataIndexer(indexerConfig, dependencies);
+                logger.log('INFO', `[PopularInvestorFetch] Root data indexer completed for date ${today}`);
+            } catch (indexerError) {
+                logger.log('ERROR', `[PopularInvestorFetch] Failed to run root data indexer for ${today}`, indexerError);
+                // Continue - rankings data is stored, indexer failure is non-critical
+            }
+            
             return { success: true, count: data.TotalRows };
 
         } catch (dbError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.475",
+  "version": "1.0.476",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [