bulltrackers-module 1.0.567 → 1.0.569

package/functions/orchestrator/helpers/update_helpers.js

@@ -42,17 +42,47 @@ async function getUpdateTargets(userType, thresholds, config, dependencies) {
 
 /**
  * Sub-pipe: pipe.orchestrator.dispatchUpdates
+ * UPDATED: Added real-time validation to skip users already updated today.
  */
 async function dispatchUpdates(targets, userType, config, dependencies) {
-    const { logger, pubsubUtils } = dependencies;
+    const { logger, pubsubUtils, db } = dependencies;
     const { dispatcherTopicName, taskBatchSize, pubsubBatchSize } = config;
     
-    if (targets.length === 0) { 
-        logger.log('INFO', `[Orchestrator Helpers] No ${userType} update targets to dispatch.`); 
-        return; 
-    }
+    if (targets.length === 0) return;
+
+    // --- NEW VALIDATION BLOCK ---
+    const startOfToday = new Date();
+    startOfToday.setUTCHours(0, 0, 0, 0);
+    
+    logger.log('INFO', `[Orchestrator Validation] Verifying ${targets.length} targets for ${userType}...`);
     
-    logger.log('INFO', `[Orchestrator Helpers] Dispatching ${targets.length} update tasks for ${userType} to ${dispatcherTopicName}...`);
+    const validTargets = [];
+    for (const target of targets) {
+        const cid = target.userId || target.cid || (typeof target === 'string' ? target : null);
+        if (!cid) { validTargets.push(target); continue; }
+
+        // Check the actual Firestore record for today's completion
+        const collection = userType === 'popular_investor' ? 'PopularInvestors' : 
+                           userType === 'signed_in_user' ? 'SignedInUsers' : 'NormalUsers';
+        
+        const doc = await db.collection(collection).doc(String(cid)).get();
+        const lastUpdated = doc.data()?.lastUpdated?.updatedAt?.toDate?.() || 
+                           doc.data()?.lastUpdate?.toDate?.();
+
+        if (lastUpdated && lastUpdated >= startOfToday) {
+            logger.log('TRACE', `[Orchestrator Validation] Skipping ${cid} - Already updated today.`);
+            continue;
+        }
+        validTargets.push(target);
+    }
+
+    if (validTargets.length === 0) {
+        logger.log('INFO', `[Orchestrator Helpers] All ${targets.length} targets already updated. Skipping dispatch.`);
+        return;
+    }
+    // --- END VALIDATION BLOCK ---
+
+    logger.log('INFO', `[Orchestrator Helpers] Dispatching ${validTargets.length} validated update tasks for ${userType}...`);
     
     const individualTasks = targets.map(target => {
         let task = { userType };

package/functions/task-engine/handler_creator.js

@@ -133,7 +133,7 @@ async function handleRequest(message, context, configObj, dependencies) {
             if (updateTasksCount > 0) {
                 try {
                     // This batch counter is critical for triggering the Root Data Indexer
-                    batchCounterRef = db.collection('task_engine_batch_counters').doc(`${today}-${taskId}`);
+                    batchCounterRef = db.collection('system_task_counts').doc(`${today}-${taskId}`);
                     await batchCounterRef.set({
                         totalTasks: updateTasksCount,
                         remainingTasks: updateTasksCount,

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -308,7 +308,7 @@ async function handleGenericUserUpdate(taskData, config, dependencies, isPI) {
         
         // 5. Batch Counter Decrement (Critical for Scheduled Runs)
         if (batchCounterId) {
-             const counterRef = db.collection('task_engine_batch_counters').doc(batchCounterId);
+             const counterRef = db.collection('system_task_counts').doc(batchCounterId);
              await counterRef.update({ remainingTasks: FieldValue.increment(-1) });
              
              // Check if we need to trigger root indexer for the batch

package/functions/task-engine/utils/task_engine_utils.js

@@ -2,6 +2,7 @@
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
  * (REFACTORED: Concurrency limit increased to 5)
  * FIXED: Increased concurrency to prevent timeouts on large batches.
+ * FIXED: Added support for SIGNED_IN_USER_UPDATE in batch execution loop.
  */
 
 /**
@@ -45,7 +46,7 @@ async function prepareTaskBatches(tasks, batchManager, logger) {
             // Social fetch tasks
             socialTasks.push(task);
         } else {
-            // Discover, Verify, Popular Investor (POPULAR_INVESTOR_UPDATE), Signed-In User (ON_DEMAND_USER_UPDATE)
+            // Discover, Verify, Popular Investor (POPULAR_INVESTOR_UPDATE), Signed-In User (ON_DEMAND_USER_UPDATE / SIGNED_IN_USER_UPDATE)
             otherTasks.push(task); 
         }
     }
@@ -90,19 +91,20 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
             continue;
         }
 
-        if (task.type === 'ON_DEMAND_USER_UPDATE') {
+        // --- FIXED: Added SIGNED_IN_USER_UPDATE check ---
+        if (task.type === 'ON_DEMAND_USER_UPDATE' || task.type === 'SIGNED_IN_USER_UPDATE') {
             const taskData = task.data || task;
             if (!taskData.cid || !taskData.username) {
-                logger.log('ERROR', `[TaskEngine/${taskId}] ON_DEMAND_USER_UPDATE task missing required fields`, { task, taskData });
+                logger.log('ERROR', `[TaskEngine/${taskId}] ${task.type} task missing required fields`, { task, taskData });
                 taskCounters.failed++;
                 continue;
             }
-            logger.log('INFO', `[TaskEngine/${taskId}] Processing ON_DEMAND_USER_UPDATE for CID: ${taskData.cid}, Username: ${taskData.username}`);
+            logger.log('INFO', `[TaskEngine/${taskId}] Processing ${task.type} for CID: ${taskData.cid}, Username: ${taskData.username}`);
             allTaskPromises.push(limit(() => 
                 handleOnDemandUserUpdate(taskData, config, dependencies)
                     .then(() => taskCounters.on_demand++)
                     .catch(err => {
-                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in ON_DEMAND_USER_UPDATE for ${taskData.cid || taskData.username}`, { errorMessage: err.message });
+                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${taskData.cid || taskData.username}`, { errorMessage: err.message });
                         taskCounters.failed++;
                     })
             ));
@@ -111,7 +113,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
 
         // Handle legacy types (discover/verify)
         const normalizedType = task.type.toLowerCase();
-        const handler = { discover:                 handleDiscover, verify: handleVerify }[normalizedType];
+        const handler = { discover: handleDiscover, verify: handleVerify }[normalizedType];
 
         if (handler) {
             allTaskPromises.push(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.567",
+  "version": "1.0.569",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/task-engine/taskengine_workflow.yaml

@@ -1,83 +0,0 @@
-# bulltrackers-module/workflows/daily_update_pipeline.yaml
-# Cloud Workflows: Slow-Trickle Daily Update Orchestrator
-# Triggers the Orchestrator to PLAN updates, then EXECUTES them in timed windows.
-
-main:
-  params: [input]
-  steps:
-    - init:
-        assign:
-          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
-          - location: "europe-west1"
-          # Replace with your actual Orchestrator HTTP trigger URL
-          - orchestrator_url: '${"https://" + location + "-" + project + ".cloudfunctions.net/orchestrator-http"}' 
-          - today: '${text.split(time.format(sys.now()), "T")[0]}'
-          # User types to process (can be passed in input or defaulted)
-          - user_types: '${default(map.get(input, "userTypes"), ["normal", "speculator"])}'
-          - default_windows: 10
-
-    - process_user_types_loop:
-        for:
-          value: user_type
-          in: ${user_types}
-          steps:
-            - log_start:
-                call: sys.log
-                args:
-                  text: '${"Starting update cycle for: " + user_type}'
-
-            # --- PHASE 1: PLAN ---
-            - plan_updates:
-                call: http.post
-                args:
-                  url: '${orchestrator_url}'
-                  body:
-                    action: 'PLAN'
-                    userType: '${user_type}'
-                    date: '${today}'
-                    windows: '${default_windows}'
-                  auth: { type: OIDC }
-                  timeout: 300 # 5 minutes to query and split users
-                result: plan_res
-
-            - log_plan:
-                call: sys.log
-                args:
-                  text: '${"📅 PLAN CREATED: " + user_type + " | PlanID: " + plan_res.body.planId + " | Users: " + plan_res.body.totalUsers + " | Windows: " + plan_res.body.windowCount}'
-
-            # --- PHASE 2: EXECUTE WINDOWS ---
-            - run_windows_loop:
-                for:
-                  value: window_id
-                  in: '${plan_res.body.windowIds}'
-                  steps:
-                    - execute_window:
-                        call: http.post
-                        args:
-                          url: '${orchestrator_url}'
-                          body:
-                            action: 'EXECUTE_WINDOW'
-                            planId: '${plan_res.body.planId}'
-                            windowId: '${window_id}'
-                            userType: '${user_type}'
-                          auth: { type: OIDC }
-                        result: exec_res
-                    
-                    - log_execution:
-                        call: sys.log
-                        args:
-                           text: '${"🚀 WINDOW EXECUTED: " + user_type + " Window " + window_id + "/" + plan_res.body.windowCount + ". Dispatched: " + exec_res.body.dispatchedCount}'
-
-                    # --- PACING: Sleep between windows ---
-                    # We skip the sleep after the very last window of the loop
-                    - check_pacing_needed:
-                        switch:
-                          - condition: '${window_id < plan_res.body.windowCount}'
-                            steps:
-                              - wait_pacing:
-                                  call: sys.sleep
-                                  args:
-                                    seconds: 3600 # 1 Hour wait between blocks
-
-    - finish:
-        return: "Daily Slow-Trickle Update Completed."