bulltrackers-module 1.0.475 → 1.0.477

package/functions/computation-system/helpers/on_demand_helpers.js

@@ -84,7 +84,10 @@ function resolveDependencyChain(computationName, manifest) {
  */
 async function triggerComputationWithDependencies(targetComputation, date, dependencies, config, metadata = {}) {
     const { pubsub, logger } = dependencies;
-    const computationTopic = config.computationTopicStandard || 'computation-tasks';
+    // Use on-demand topic for on-demand computation requests
+    const computationTopic = config.computationTopicOnDemand || 
+                            (config.getComputationTopic && config.getComputationTopic(true)) ||
+                            'computation-tasks-ondemand';
     const topic = pubsub.topic(computationTopic);
     const crypto = require('crypto');
     

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,6 +1,7 @@
-# Data Feeder Pipeline (V3.1 - Syntax Fixed)
+# Data Feeder Pipeline (V3.2 - Automatic Root Data Indexing)
 # Starts at 22:00 UTC via Cloud Scheduler.
-# Fixes: Split assign/call steps and corrected assign syntax.
+# UPDATED: Removed intermediate root data indexer calls - each fetcher now automatically triggers indexing.
+# Only the global verification run at midnight remains.
 
 main:
   params: [input]
@@ -45,17 +46,7 @@ main:
     - wait_10_after_price:
         call: sys.sleep
         args: { seconds: 600 } # 10 Minutes
-
-    # FIX 1: Split assign and call
-    - prepare_index_price:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_price:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Price fetcher now automatically triggers root data indexer after completion
 
     - wait_10_before_insights:
         call: sys.sleep
@@ -78,17 +69,7 @@ main:
     - wait_10_after_insights:
         call: sys.sleep
         args: { seconds: 600 }
-
-    # FIX 2: Split assign and call
-    - prepare_index_insights:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_insights:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Insights fetcher now automatically triggers root data indexer after completion
 
     # ==========================================
     # PHASE 2: WAIT FOR MIDNIGHT
@@ -124,17 +105,7 @@ main:
     - wait_10_after_rankings:
         call: sys.sleep
         args: { seconds: 600 }
-
-    # FIX 3: Split assign and call
-    - prepare_index_rankings:
-        assign:
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-    - index_today_after_rankings:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body: { targetDate: '${today}' }
-          auth: { type: OIDC }
+        # NOTE: Popular investor rankings fetcher now automatically triggers root data indexer after completion
 
     - phase_0000_social:
         try:
@@ -158,7 +129,7 @@ main:
         call: http.post
         args:
           url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          # No targetDate = Global Run
+          # No targetDate = Global verification run (all intermediate indexing is now automatic)
           auth: { type: OIDC }
 
     # ==========================================
@@ -194,17 +165,7 @@ main:
               - wait_10_in_loop:
                   call: sys.sleep
                   args: { seconds: 600 }
-              
-              # FIX 4: Split assign and call
-              - prepare_index_loop:
-                   assign:
-                     - today: '${text.split(time.format(sys.now()), "T")[0]}'
-              - index_today_in_loop:
-                   call: http.post
-                   args:
-                     url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-                     body: { targetDate: '${today}' }
-                     auth: { type: OIDC }
+                  # NOTE: Social tasks are handled by task engine, which automatically triggers root data indexer after batch completion
               
               # FIX 5: Correct assign syntax (must be a list)
               - increment_loop:

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -113,6 +113,37 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
             });
 
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${rankingsCollectionName}/${today}`);
+            
+            // Update root data indexer for today's date after rankings data is stored
+            try {
+                const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                // Access rootDataIndexer from config (passed from index.js) or use defaults
+                // Using bracket notation to avoid TypeScript errors
+                let rootDataIndexerConfig;
+                if (config && typeof config === 'object' && config['rootDataIndexer']) {
+                    rootDataIndexerConfig = config['rootDataIndexer'];
+                } else {
+                    rootDataIndexerConfig = {
+                        availabilityCollection: 'system_root_data_index',
+                        earliestDate: '2025-08-01',
+                        collections: {
+                            piRankings: rankingsCollectionName
+                        }
+                    };
+                }
+                
+                const indexerConfig = Object.assign({}, rootDataIndexerConfig, {
+                    targetDate: today // Index only today's date for speed
+                });
+                
+                logger.log('INFO', `[PopularInvestorFetch] Triggering root data indexer for date ${today} after rankings data storage...`);
+                await runRootDataIndexer(indexerConfig, dependencies);
+                logger.log('INFO', `[PopularInvestorFetch] Root data indexer completed for date ${today}`);
+            } catch (indexerError) {
+                logger.log('ERROR', `[PopularInvestorFetch] Failed to run root data indexer for ${today}`, indexerError);
+                // Continue - rankings data is stored, indexer failure is non-critical
+            }
+            
             return { success: true, count: data.TotalRows };
 
         } catch (dbError) {

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -134,8 +134,10 @@ async function requestPiFetch(req, res, dependencies, config) {
             updatedAt: FieldValue.serverTimestamp()
         }, { merge: true });
         
-        // Publish to task engine
-        const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH || 'etoro-user-fetch-topic';
+        // Publish to task engine - use on-demand topic for API requests
+        const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH_ONDEMAND || 
+                         config.pubsubTopicUserFetchOnDemand || 
+                         'etoro-user-fetch-topic-ondemand';
         const topic = pubsub.topic(topicName);
         
         const message = {

package/functions/generic-api/user-api/helpers/user_sync_helpers.js

@@ -148,8 +148,10 @@ async function requestUserSync(req, res, dependencies, config) {
             updatedAt: FieldValue.serverTimestamp()
         }, { merge: true });
         
-        // Publish to task engine
-        const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH || 'etoro-user-fetch-topic';
+        // Publish to task engine - use on-demand topic for API requests
+        const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH_ONDEMAND || 
+                         config.pubsubTopicUserFetchOnDemand || 
+                         'etoro-user-fetch-topic-ondemand';
         const topic = pubsub.topic(topicName);
         
         const message = {

package/functions/generic-api/user-api/helpers/verification_helpers.js

@@ -57,8 +57,12 @@ async function finalizeVerification(req, res, dependencies, config) {
         proxyConfig, 
         headerConfig,
         pubsubTopicUserFetch,
+        pubsubTopicUserFetchOnDemand,
         pubsubTopicSocialFetch
     } = config;
+    
+    // Use on-demand topic for user signup (API-triggered)
+    const taskEngineTopic = pubsubTopicUserFetchOnDemand || pubsubTopicUserFetch || 'etoro-user-fetch-topic-ondemand';
 
     if (!username) return res.status(400).json({ error: "Missing username." });
 
@@ -187,8 +191,8 @@ async function finalizeVerification(req, res, dependencies, config) {
         
         // Only trigger if user is public (has portfolio data)
         if (!isOptOut) {
-            await pubsubUtils.publish(pubsubTopicUserFetch, unifiedTask);
-            logger.log('INFO', `[Verification] Triggered unified data fetch (portfolio + social) for ${username} (${realCID})`);
+            await pubsubUtils.publish(taskEngineTopic, unifiedTask);
+            logger.log('INFO', `[Verification] Triggered unified data fetch (portfolio + social) for ${username} (${realCID}) via on-demand topic`);
         } else {
             // For private users, still fetch social data but no portfolio
             const socialOnlyTask = {
@@ -208,8 +212,8 @@ async function finalizeVerification(req, res, dependencies, config) {
                     requestedAt: new Date().toISOString()
                 }
             };
-            await pubsubUtils.publish(pubsubTopicUserFetch, socialOnlyTask);
-            logger.log('INFO', `[Verification] Triggered social-only fetch for private user ${username} (${realCID})`);
+            await pubsubUtils.publish(taskEngineTopic, socialOnlyTask);
+            logger.log('INFO', `[Verification] Triggered social-only fetch for private user ${username} (${realCID}) via on-demand topic`);
         }
 
         return res.status(200).json({ 

package/functions/task-engine/handler_creator.js

@@ -13,15 +13,19 @@ const { handleSocialFetch } = require('./helpers/social_helpers');
 const { executeTasks, prepareTaskBatches } = require('./utils/task_engine_utils'); 
 
 async function handleRequest(message, context, configObj, dependencies) {
-    // Support both old format (single config) and new format (object with taskEngine, rootDataIndexer, and social)
+    // Support both old format (single config) and new format (object with taskEngine, rootDataIndexer, social, and computationSystem)
     const config = configObj.taskEngine || configObj; // Backward compatibility
     const rootDataIndexerConfig = configObj.rootDataIndexer;
     const socialConfig = configObj.social;
+    const computationSystemConfig = configObj.computationSystem;
     
-    // Merge social config into main config for easy access
+    // Merge configs into main config for easy access
     if (socialConfig) {
         config.social = socialConfig;
     }
+    if (computationSystemConfig) {
+        config.computationSystem = computationSystemConfig;
+    }
     const { logger, batchManager, db } = dependencies;
     
     // [CRITICAL FIX] Max Age increased to 25m to match the larger dedup window.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.475",
+  "version": "1.0.477",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [