bulltrackers-module 1.0.556 → 1.0.557

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,5 +1,6 @@
-# Data Feeder Pipeline (V3.3 - Optimized Waits & Isolated Testing)
-# Starts at 22:00 UTC via Cloud Scheduler.
+# Data Feeder Pipeline (V4.0 - Lean Market Close)
+# Schedule: 22:00 UTC
+# Objective: Capture Market Close data & perform Global Indexing at midnight.
 
 main:
   params: [input]
@@ -10,7 +11,7 @@ main:
           - location: "europe-west1"
 
     # ==========================================
-    # TEST MODE ROUTING
+    # TEST MODE ROUTING (Preserved for Debugging)
     # ==========================================
     - check_test_mode:
         switch:
@@ -18,9 +19,6 @@ main:
             steps:
               - route_test:
                   switch:
-                    # --- ISOLATED FUNCTION TESTS (Run & Stop) ---
-                    # Use these to test a single function without triggering waits or next steps.
-
                     - condition: '${input.target_step == "test_price"}'
                       steps:
                         - call_price_iso:
@@ -42,29 +40,7 @@ main:
                               timeout: 300
                         - return_insights:
                             return: "Test Complete: Insights Fetcher executed."
-
-                    - condition: '${input.target_step == "test_rankings"}'
-                      steps:
-                        - call_rankings_iso:
-                            call: http.post
-                            args:
-                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
-                              auth: { type: OIDC }
-                              timeout: 300
-                        - return_rankings:
-                            return: "Test Complete: Popular Investors executed."
-
-                    - condition: '${input.target_step == "test_social"}'
-                      steps:
-                        - call_social_iso:
-                            call: http.post
-                            args:
-                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
-                              auth: { type: OIDC }
-                              timeout: 300
-                        - return_social:
-                            return: "Test Complete: Social Orchestrator executed."
-
+                            
                     - condition: '${input.target_step == "test_indexer"}'
                       steps:
                         - call_indexer_iso:
@@ -76,18 +52,11 @@ main:
                         - return_indexer:
                             return: "Test Complete: Root Data Indexer executed."
 
-                    # --- PHASE JUMPS (Resumes flow from that point) ---
-                    - condition: '${input.target_step == "market"}'
-                      next: phase_2200_price
-                    - condition: '${input.target_step == "midnight"}'
-                      next: phase_0000_rankings
-                    - condition: '${input.target_step == "social"}'
-                      next: social_loop_start
-
     # ==========================================
     # PHASE 1: MARKET CLOSE (Starts 22:00 UTC)
     # ==========================================
 
+    # 1. Price Fetcher
     - phase_2200_price:
         try:
           call: http.post
@@ -101,12 +70,8 @@ main:
             - log_price_error:
                 call: sys.log
                 args: { severity: "WARNING", text: "Price fetch timed out/failed. Proceeding." }
-    
-    # FIXED: Only one 10-minute wait here now.
-    - wait_10_after_price:
-        call: sys.sleep
-        args: { seconds: 600 } 
 
+    # 2. Insights Fetcher (Can run immediately after or parallel)
     - phase_2200_insights:
         try:
           call: http.post
@@ -121,108 +86,30 @@ main:
                 call: sys.log
                 args: { severity: "WARNING", text: "Insights fetch timed out/failed. Proceeding." }
 
-    - wait_10_after_insights:
-        call: sys.sleep
-        args: { seconds: 600 }
-
     # ==========================================
-    # PHASE 2: WAIT FOR MIDNIGHT
+    # PHASE 2: WAIT FOR MIDNIGHT (Indexing)
     # ==========================================
 
     - align_to_midnight:
         assign:
           - now_sec: '${int(sys.now())}'
           - day_sec: 86400
+          # Calculates seconds remaining until the next 00:00 UTC
           - sleep_midnight: '${day_sec - (now_sec % day_sec)}'
+    
     - wait_for_midnight:
         call: sys.sleep
         args: { seconds: '${sleep_midnight}' }
 
     # ==========================================
-    # PHASE 3: MIDNIGHT TASKS (00:00 UTC)
+    # PHASE 3: GLOBAL INDEXING (00:00 UTC)
     # ==========================================
 
-    - phase_0000_rankings:
-        try:
-          call: http.post
-          args:
-            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
-            auth: { type: OIDC }
-            timeout: 300
-        except:
-          as: e
-          steps:
-            - log_ranking_error:
-                call: sys.log
-                args: { severity: "WARNING", text: "Rankings failed. Proceeding to Social (risky)." }
-
-    - wait_10_after_rankings:
-        call: sys.sleep
-        args: { seconds: 600 }
-
-    - phase_0000_social:
-        try:
-          call: http.post
-          args:
-            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
-            auth: { type: OIDC }
-            timeout: 300
-        except:
-          as: e
-          steps:
-            - log_social_error:
-                call: sys.log
-                args: { severity: "WARNING", text: "Social failed. Proceeding." }
-
-    - wait_10_after_social:
-        call: sys.sleep
-        args: { seconds: 600 }
-
     - global_index_midnight:
         call: http.post
         args:
           url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
           auth: { type: OIDC }
 
-    # ==========================================
-    # PHASE 4: SOCIAL LOOP (Every 3 Hours)
-    # ==========================================
-
-    - init_social_loop:
-        assign:
-          - i: 0
-    
-    - social_loop_start:
-        switch:
-          - condition: ${i < 7} # Covers the remainder of the 24h cycle
-            steps:
-              - wait_3_hours:
-                  call: sys.sleep
-                  args: { seconds: 10800 }
-              
-              - run_social_recurring:
-                  try:
-                    call: http.post
-                    args:
-                      url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
-                      auth: { type: OIDC }
-                      timeout: 300
-                  except:
-                    as: e
-                    steps:
-                      - log_loop_social_warn:
-                          call: sys.log
-                          args: { severity: "WARNING", text: "Loop Social timed out. Proceeding." }
-
-              - wait_10_in_loop:
-                  call: sys.sleep
-                  args: { seconds: 600 }
-              
-              - increment_loop:
-                  assign: 
-                    - i: '${i + 1}'
-              - next_iteration:
-                  next: social_loop_start
-
     - finish:
-        return: "Complete 24h Cycle Finished"
+        return: "Daily Close Cycle Finished (Price, Insights, Indexing)."

package/functions/computation-system/workflows/morning_prep_pipeline.yaml

@@ -0,0 +1,55 @@
+# Morning Prep Pipeline
+# Schedule: 04:00 UTC
+# Objective: Refresh Rankings -> Trigger Daily Update (Task Engine)
+
+main:
+  params: [input]
+  steps:
+    - init:
+        assign:
+          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
+          - location: "europe-west1"
+          # Define the Task Engine workflow ID to trigger later
+          - task_engine_workflow_id: "daily-update-pipeline"
+
+    # ==========================================
+    # STEP 1: REFRESH RANKINGS (The Master List)
+    # ==========================================
+    - fetch_rankings:
+        try:
+          call: http.post
+          args:
+            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
+            auth: { type: OIDC }
+            timeout: 540
+        except:
+          as: e
+          steps:
+            - log_rankings_fail:
+                call: sys.log
+                args: { severity: "ERROR", text: "Rankings Fetch Failed. Stopping pipeline to prevent stale Task Engine run." }
+            - raise_error:
+                raise: ${e}
+
+    # ==========================================
+    # STEP 2: TRIGGER TASK ENGINE
+    # ==========================================
+    # We trigger the workflow execution directly. 
+    # This replaces the need for a separate 05:00 UTC Scheduler.
+    
+    - trigger_daily_update:
+        call: googleapis.workflowexecutions.v1.projects.locations.workflows.executions.create
+        args:
+          parent: '${"projects/" + project + "/locations/" + location + "/workflows/" + task_engine_workflow_id}'
+          body:
+            # Pass explicit arguments to the Task Engine
+            argument: '{"userTypes": ["normal", "speculator"], "source": "morning_prep_trigger"}'
+        result: execution_result
+
+    - log_trigger:
+        call: sys.log
+        args:
+          text: '${"✅ Rankings Complete. Triggered Task Engine: " + execution_result.name}'
+
+    - finish:
+        return: "Morning Prep Complete. Task Engine launched."

package/functions/orchestrator/index.js

@@ -2,10 +2,183 @@
  * @fileoverview Main orchestration logic.
  * REFACTORED: This file now contains the main pipe functions
  * that are called by the Cloud Function entry points.
+ * It includes the new HTTP handlers for Workflow-driven "Slow-Trickle" updates.
  * They receive all dependencies.
  */
 const { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery } = require('./helpers/discovery_helpers');
 const { getUpdateTargets, dispatchUpdates } = require('./helpers/update_helpers');
+const { FieldValue } = require('@google-cloud/firestore');
+
+/**
+ * ENTRY POINT: HTTP Handler for Workflow Interaction
+ * Map this function to your HTTP Trigger in your index.js/exports.
+ * This handles the "PLAN" and "EXECUTE_WINDOW" phases of the slow-trickle update.
+ * * @param {object} req - Express request object.
+ * @param {object} res - Express response object.
+ * @param {object} dependencies - Contains logger, db, firestoreUtils, pubsubUtils.
+ * @param {object} config - Global configuration.
+ */
+async function handleOrchestratorHttp(req, res, dependencies, config) {
+    const { logger } = dependencies;
+    const body = req.body || {};
+    const { action, userType, date, windows, planId, windowId } = body;
+
+    logger.log('INFO', `[Orchestrator HTTP] Received request: ${action}`, body);
+
+    try {
+        if (action === 'PLAN') {
+            // PHASE 1: Find users and split them into Firestore documents
+            if (!userType || !date) {
+                throw new Error("Missing userType or date for PLAN action");
+            }
+            const result = await planDailyUpdates(userType, date, windows || 10, config, dependencies);
+            res.status(200).send(result);
+
+        } else if (action === 'EXECUTE_WINDOW') {
+            // PHASE 2: Load specific window and dispatch
+            if (!planId || !windowId) {
+                throw new Error("Missing planId or windowId for EXECUTE_WINDOW action");
+            }
+            const result = await executeUpdateWindow(planId, windowId, userType, config, dependencies);
+            res.status(200).send(result);
+
+        } else if (action === 'LEGACY_RUN') {
+            // Support for triggering the old brute-force method via HTTP if needed
+            await runUpdateOrchestrator(config, dependencies);
+            res.status(200).send({ status: 'Completed legacy run' });
+
+        } else {
+            res.status(400).send({ error: `Unknown action: ${action}` });
+        }
+    } catch (error) {
+        logger.log('ERROR', `[Orchestrator HTTP] Fatal error in ${action}`, { errorMessage: error.message, stack: error.stack });
+        res.status(500).send({ error: error.message, stack: error.stack });
+    }
+}
+
+/**
+ * LOGIC: Plan the updates (Split into windows)
+ * 1. Fetches all users needing updates.
+ * 2. Shuffles them.
+ * 3. Splits them into 'n' windows.
+ * 4. Saves the windows to Firestore.
+ */
+async function planDailyUpdates(userType, date, numberOfWindows, config, deps) {
+    const { logger, db } = deps;
+    
+    // 1. Get ALL targets
+    // We construct thresholds to capture everyone due for today
+    const now = new Date();
+    const startOfTodayUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
+    const DaysAgoUTC = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
+    
+    const thresholds = { 
+        dateThreshold: startOfTodayUTC, 
+        gracePeriodThreshold: DaysAgoUTC 
+    };
+
+    logger.log('INFO', `[Orchestrator Plan] Calculating targets for ${userType}...`);
+
+    // Reusing existing helper to get the raw list of users
+    const targets = await getUpdateTargets(userType, thresholds, config.updateConfig, deps);
+    logger.log('INFO', `[Orchestrator Plan] Found ${targets.length} candidates for ${userType}.`);
+
+    if (targets.length === 0) {
+        return { planId: null, totalUsers: 0, windowCount: 0, windowIds: [] };
+    }
+
+    // 2. Shuffle to randomize load (Fisher-Yates shuffle)
+    // This ensures that we don't always update the same users at the same time of day
+    for (let i = targets.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [targets[i], targets[j]] = [targets[j], targets[i]];
+    }
+
+    // 3. Split and Save
+    const chunkSize = Math.ceil(targets.length / numberOfWindows);
+    const planId = `plan_${userType}_${date}`;
+    const windowIds = [];
+
+    const batchWriter = db.batch();
+    let writeCount = 0;
+    
+    for (let i = 0; i < numberOfWindows; i++) {
+        const start = i * chunkSize;
+        const end = start + chunkSize;
+        const chunk = targets.slice(start, end);
+
+        if (chunk.length > 0) {
+            // Store ONLY the necessary IDs/Data in Firestore
+            // Path: system_update_plans/{planId}/windows/{windowId}
+            const windowDocRef = db.collection('system_update_plans').doc(planId).collection('windows').doc(String(i + 1));
+            
+            batchWriter.set(windowDocRef, {
+                userType: userType,
+                users: chunk, // This array contains the user objects/IDs
+                status: 'pending',
+                windowId: i + 1,
+                userCount: chunk.length,
+                createdAt: FieldValue.serverTimestamp(),
+                scheduledForDate: date
+            });
+            
+            windowIds.push(i + 1);
+            writeCount++;
+        }
+    }
+
+    await batchWriter.commit();
+    logger.log('SUCCESS', `[Orchestrator Plan] Plan Saved: ${planId} with ${writeCount} windows containing ${targets.length} users.`);
+
+    return {
+        planId: planId,
+        totalUsers: targets.length,
+        windowCount: windowIds.length,
+        windowIds: windowIds
+    };
+}
+
+/**
+ * LOGIC: Execute a specific window
+ * 1. Reads the user list from Firestore.
+ * 2. Calls dispatchUpdates to send them to the Task Engine.
+ */
+async function executeUpdateWindow(planId, windowId, userType, config, deps) {
+    const { logger, db } = deps;
+
+    // 1. Fetch the window from Firestore
+    const windowRef = db.collection('system_update_plans').doc(planId).collection('windows').doc(String(windowId));
+    const windowDoc = await windowRef.get();
+
+    if (!windowDoc.exists) {
+        throw new Error(`Window ${windowId} not found in plan ${planId}`);
+    }
+
+    const data = windowDoc.data();
+    
+    // Idempotency check: prevent re-running a completed window
+    if (data.status === 'completed') {
+        logger.log('WARN', `[Orchestrator Execute] Window ${windowId} already completed. Skipping.`);
+        return { dispatchedCount: 0, status: 'already_completed' };
+    }
+
+    const targets = data.users;
+    logger.log('INFO', `[Orchestrator Execute] Window ${windowId}: Dispatching ${targets.length} users.`);
+
+    // 2. Dispatch using existing helper
+    // The helper handles batching for Pub/Sub and logging.
+    if (targets && targets.length > 0) {
+        await dispatchUpdates(targets, userType, config.updateConfig, deps);
+    }
+
+    // 3. Mark window as complete
+    await windowRef.update({ 
+        status: 'completed',
+        executedAt: FieldValue.serverTimestamp()
+    });
+
+    return { dispatchedCount: targets.length, status: 'success' };
+}
 
 /** Stage 1: Main discovery orchestrator pipe */
 async function runDiscoveryOrchestrator(config, deps) {
@@ -135,4 +308,11 @@ function isUserTypeEnabled(userType, enabledTypes) {
     return enabledTypes.includes(userType);
 }
 
-module.exports = { runDiscoveryOrchestrator, runUpdateOrchestrator, runDiscovery, runUpdates, isUserTypeEnabled };
+module.exports = { 
+    handleOrchestratorHttp,
+    runDiscoveryOrchestrator, 
+    runUpdateOrchestrator, 
+    runDiscovery, 
+    runUpdates, 
+    isUserTypeEnabled 
+};

package/functions/task-engine/taskengine_workflow.yaml

@@ -0,0 +1,83 @@
+# bulltrackers-module/workflows/daily_update_pipeline.yaml
+# Cloud Workflows: Slow-Trickle Daily Update Orchestrator
+# Triggers the Orchestrator to PLAN updates, then EXECUTES them in timed windows.
+
+main:
+  params: [input]
+  steps:
+    - init:
+        assign:
+          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
+          - location: "europe-west1"
+          # Replace with your actual Orchestrator HTTP trigger URL
+          - orchestrator_url: '${"https://" + location + "-" + project + ".cloudfunctions.net/orchestrator-http"}' 
+          - today: '${text.split(time.format(sys.now()), "T")[0]}'
+          # User types to process (can be passed in input or defaulted)
+          - user_types: '${default(map.get(input, "userTypes"), ["normal", "speculator"])}'
+          - default_windows: 10
+
+    - process_user_types_loop:
+        for:
+          value: user_type
+          in: ${user_types}
+          steps:
+            - log_start:
+                call: sys.log
+                args:
+                  text: '${"Starting update cycle for: " + user_type}'
+
+            # --- PHASE 1: PLAN ---
+            - plan_updates:
+                call: http.post
+                args:
+                  url: '${orchestrator_url}'
+                  body:
+                    action: 'PLAN'
+                    userType: '${user_type}'
+                    date: '${today}'
+                    windows: '${default_windows}'
+                  auth: { type: OIDC }
+                  timeout: 300 # 5 minutes to query and split users
+                result: plan_res
+
+            - log_plan:
+                call: sys.log
+                args:
+                  text: '${"📅 PLAN CREATED: " + user_type + " | PlanID: " + plan_res.body.planId + " | Users: " + plan_res.body.totalUsers + " | Windows: " + plan_res.body.windowCount}'
+
+            # --- PHASE 2: EXECUTE WINDOWS ---
+            - run_windows_loop:
+                for:
+                  value: window_id
+                  in: '${plan_res.body.windowIds}'
+                  steps:
+                    - execute_window:
+                        call: http.post
+                        args:
+                          url: '${orchestrator_url}'
+                          body:
+                            action: 'EXECUTE_WINDOW'
+                            planId: '${plan_res.body.planId}'
+                            windowId: '${window_id}'
+                            userType: '${user_type}'
+                          auth: { type: OIDC }
+                        result: exec_res
+                    
+                    - log_execution:
+                        call: sys.log
+                        args:
+                           text: '${"🚀 WINDOW EXECUTED: " + user_type + " Window " + window_id + "/" + plan_res.body.windowCount + ". Dispatched: " + exec_res.body.dispatchedCount}'
+
+                    # --- PACING: Sleep between windows ---
+                    # We skip the sleep after the very last window of the loop
+                    - check_pacing_needed:
+                        switch:
+                          - condition: '${window_id < plan_res.body.windowCount}'
+                            steps:
+                              - wait_pacing:
+                                  call: sys.sleep
+                                  args:
+                                    seconds: 3600 # 1 Hour wait between blocks
+
+    - finish:
+        return: "Daily Slow-Trickle Update Completed."

package/index.js

@@ -11,7 +11,13 @@ const { FirestoreBatchManager }    = require('./functions/task-engine/utils/fire
 const firestoreUtils               = require('./functions/core/utils/firestore_utils');
 
 // Orchestrator
-const { runDiscoveryOrchestrator, runUpdateOrchestrator }               = require('./functions/orchestrator/index');
+// [UPDATED] Imported handleOrchestratorHttp
+const { 
+    runDiscoveryOrchestrator, 
+    runUpdateOrchestrator, 
+    handleOrchestratorHttp 
+} = require('./functions/orchestrator/index');
+
 const { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery } = require('./functions/orchestrator/helpers/discovery_helpers');
 const { getUpdateTargets, dispatchUpdates }                             = require('./functions/orchestrator/helpers/update_helpers');
 
@@ -72,6 +78,8 @@ const core = {
 };
 
 const orchestrator = {
+  // [UPDATED] Exported handleOrchestratorHttp so it can be mapped in Cloud Functions
+  handleOrchestratorHttp,
   runDiscoveryOrchestrator,
   runUpdateOrchestrator,
   checkDiscoveryNeed,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.556",
+  "version": "1.0.557",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system/scripts/force_run.js

@@ -1,72 +0,0 @@
-/**
- * Force Run Control Script
- * Usage: 
- * node scripts/force_run.js <ComputationName> [YYYY-MM-DD]
- * * Examples:
- * node scripts/force_run.js TestSystemProbe 2026-01-02  (Runs specific day)
- * node scripts/force_run.js TestSystemProbe             (Runs ALL days from start)
- */
-
-require('dotenv').config(); // Load environment variables
-const { PubSub } = require('@google-cloud/pubsub');
-
-// CONFIGURATION
-const PROJECT_ID = process.env.GCP_PROJECT_ID || 'stocks-12345';
-const TOPIC_NAME = process.env.PUBSUB_TOPIC_DISPATCH || 'dispatch-topic'; // Must match orchestrator_config.js
-const SYSTEM_START_DATE = '2026-01-01'; // The beginning of time for your system
-
-async function main() {
-  const args = process.argv.slice(2);
-  const computationName = args[0];
-  const specificDate = args[1];
-
-  if (!computationName) {
-    console.error('❌ Error: Computation Name is required.');
-    console.log('Usage: node scripts/force_run.js <ComputationName> [YYYY-MM-DD]');
-    process.exit(1);
-  }
-
-  const pubsub = new PubSub({ projectId: PROJECT_ID });
-  const topic = pubsub.topic(TOPIC_NAME);
-
-  // 1. Logic: Single Date vs. All Dates
-  if (specificDate) {
-    await triggerComputation(topic, computationName, specificDate);
-  } else {
-    console.log(`⚠️  No date provided. Triggering REPLAY for ${computationName} from ${SYSTEM_START_DATE} to Today...`);
-    
-    let currentDate = new Date(SYSTEM_START_DATE);
-    const today = new Date();
-
-    while (currentDate <= today) {
-      const dateStr = currentDate.toISOString().split('T')[0];
-      await triggerComputation(topic, computationName, dateStr);
-      
-      // Move to next day
-      currentDate.setDate(currentDate.getDate() + 1);
-      
-      // Small throttle to prevent flooding Pub/Sub quota if years of data
-      await new Promise(r => setTimeout(r, 100)); 
-    }
-  }
-}
-
-async function triggerComputation(topic, computation, date) {
-  const payload = {
-    computationName: computation,
-    date: date,
-    force: true,      // Tells Orchestrator to ignore "Already Complete" status
-    source: 'manual-cli' // Audit trail
-  };
-
-  const dataBuffer = Buffer.from(JSON.stringify(payload));
-
-  try {
-    const messageId = await topic.publishMessage({ data: dataBuffer });
-    console.log(`✅ [${date}] Triggered ${computation} (Msg ID: ${messageId})`);
-  } catch (error) {
-    console.error(`❌ [${date}] Failed to trigger ${computation}:`, error.message);
-  }
-}
-
-main().catch(console.error);