bulltrackers-module 1.0.721 → 1.0.722

package/functions/orchestrator/index.js

@@ -1,41 +1,44 @@
 /**
  * @fileoverview Main orchestration logic.
- * REFACTORED: This file now contains the main pipe functions
- * that are called by the Cloud Function entry points.
- * It includes the new HTTP handlers for Workflow-driven "Slow-Trickle" updates.
- * They receive all dependencies.
+ * REFACTORED: Implements 'Active Scheduling' via Cloud Tasks to enforce 1-hour gaps.
+ * This allows the Workflow to finish immediately while Cloud Tasks manages the pacing.
  */
 const { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery } = require('./helpers/discovery_helpers');
 const { getUpdateTargets, dispatchUpdates } = require('./helpers/update_helpers');
 const { FieldValue } = require('@google-cloud/firestore');
+const { CloudTasksClient } = require('@google-cloud/tasks');
+
+// Initialize Cloud Tasks Client
+const cloudTasksClient = new CloudTasksClient();
 
 /**
  * ENTRY POINT: HTTP Handler for Workflow Interaction
- * Map this function to your HTTP Trigger in your index.js/exports.
- * This handles the "PLAN" and "EXECUTE_WINDOW" phases of the slow-trickle update.
- * * @param {object} req - Express request object.
- * @param {object} res - Express response object.
- * @param {object} dependencies - Contains logger, db, firestoreUtils, pubsubUtils.
- * @param {object} config - Global configuration.
  */
 async function handleOrchestratorHttp(req, res, dependencies, config) {
     const { logger } = dependencies;
     const body = req.body || {};
-    const { action, userType, date, windows, planId, windowId } = body;
+    const { action, userType, date, windows, planId, windowId, orchestratorUrlOverride } = body;
 
     logger.log('INFO', `[Orchestrator HTTP] Received request: ${action}`, body);
 
     try {
         if (action === 'PLAN') {
-            // PHASE 1: Find users and split them into Firestore documents
+            // PHASE 1: Plan AND Schedule
             if (!userType || !date) {
                 throw new Error("Missing userType or date for PLAN action");
             }
-            const result = await planDailyUpdates(userType, date, windows || 10, config, dependencies);
+            
+            // Determine self-URL for callback
+            const project = process.env.GCP_PROJECT || process.env.GOOGLE_CLOUD_PROJECT;
+            const location = process.env.GCP_LOCATION || 'us-central1'; // Default to us-central1 if not set
+            // Fallback URL construction or use passed override
+            const orchestratorUrl = orchestratorUrlOverride || `https://${location}-${project}.cloudfunctions.net/orchestrator-http`;
+            
+            const result = await planDailyUpdates(userType, date, windows || 10, config, dependencies, orchestratorUrl);
             res.status(200).send(result);
 
         } else if (action === 'EXECUTE_WINDOW') {
-            // PHASE 2: Load specific window and dispatch
+            // PHASE 2: Execute (Called by Cloud Tasks)
             if (!planId || !windowId) {
                 throw new Error("Missing planId or windowId for EXECUTE_WINDOW action");
             }
@@ -43,7 +46,6 @@ async function handleOrchestratorHttp(req, res, dependencies, config) {
             res.status(200).send(result);
 
         } else if (action === 'LEGACY_RUN') {
-            // Support for triggering the old brute-force method via HTTP if needed
             await runUpdateOrchestrator(config, dependencies);
             res.status(200).send({ status: 'Completed legacy run' });
 
@@ -57,96 +59,141 @@ async function handleOrchestratorHttp(req, res, dependencies, config) {
 }
 
 /**
- * LOGIC: Plan the updates (Split into windows)
+ * LOGIC: Plan the updates (Split into windows AND Schedule Execution)
  * 1. Fetches all users needing updates.
  * 2. Shuffles them.
  * 3. Splits them into 'n' windows.
- * 4. Saves the windows to Firestore.
+ * 4. Schedules Cloud Tasks for each window with a 1-hour delay between them.
  */
-async function planDailyUpdates(userType, date, numberOfWindows, config, deps) {
+async function planDailyUpdates(userType, date, numberOfWindows, config, deps, orchestratorUrl) {
     const { logger, db } = deps;
     
+    // Cloud Tasks Config
+    const project = process.env.GCP_PROJECT || process.env.GOOGLE_CLOUD_PROJECT;
+    const location = process.env.GCP_LOCATION || 'us-central1'; 
+    const queue = process.env.GCP_QUEUE_NAME || 'orchestrator-queue'; // MUST EXIST in Cloud Console
+
+    // Construct the fully qualified queue name
+    const parentQueue = cloudTasksClient.queuePath(project, location, queue);
+
     // 1. Get ALL targets
-    // We construct thresholds to capture everyone due for today
     const now = new Date();
     const startOfTodayUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
-    const DaysAgoUTC = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
-    
     const thresholds = { 
         dateThreshold: startOfTodayUTC, 
-        gracePeriodThreshold: DaysAgoUTC 
+        gracePeriodThreshold: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000) 
     };
 
     logger.log('INFO', `[Orchestrator Plan] Calculating targets for ${userType}...`);
-
-    // Reusing existing helper to get the raw list of users
     const targets = await getUpdateTargets(userType, thresholds, config.updateConfig, deps);
-    logger.log('INFO', `[Orchestrator Plan] Found ${targets.length} candidates for ${userType}.`);
 
     if (targets.length === 0) {
-        return { planId: null, totalUsers: 0, windowCount: 0, windowIds: [] };
+        return { planId: null, message: `No targets found for ${userType}` };
     }
 
-    // 2. Shuffle to randomize load (Fisher-Yates shuffle)
-    // This ensures that we don't always update the same users at the same time of day
+    // 2. Shuffle (Fisher-Yates)
     for (let i = targets.length - 1; i > 0; i--) {
         const j = Math.floor(Math.random() * (i + 1));
         [targets[i], targets[j]] = [targets[j], targets[i]];
     }
 
-    // 3. Split and Save
+    // 3. Split, Save, and Schedule
     const chunkSize = Math.ceil(targets.length / numberOfWindows);
     const planId = `plan_${userType}_${date}`;
-    const windowIds = [];
-
     const batchWriter = db.batch();
-    let writeCount = 0;
     
+    // We collect task creation promises to await them at the end
+    const tasksToCreate = [];
+
     for (let i = 0; i < numberOfWindows; i++) {
         const start = i * chunkSize;
         const end = start + chunkSize;
         const chunk = targets.slice(start, end);
 
         if (chunk.length > 0) {
-            // Store ONLY the necessary IDs/Data in Firestore
-            // Path: system_update_plans/{planId}/windows/{windowId}
-            const windowDocRef = db.collection('system_update_plans').doc(planId).collection('windows').doc(String(i + 1));
+            const windowId = i + 1;
+            
+            // A. Prepare Firestore Write
+            const windowDocRef = db.collection('system_update_plans').doc(planId).collection('windows').doc(String(windowId));
             
             batchWriter.set(windowDocRef, {
                 userType: userType,
-                users: chunk, // This array contains the user objects/IDs
+                users: chunk, 
                 status: 'pending',
-                windowId: i + 1,
+                windowId: windowId,
                 userCount: chunk.length,
                 createdAt: FieldValue.serverTimestamp(),
                 scheduledForDate: date
             });
             
-            windowIds.push(i + 1);
-            writeCount++;
+            // B. Prepare Cloud Task
+            // Delay Logic: Window 1 = 0s, Window 2 = 3600s (1hr), Window 3 = 7200s (2hr)...
+            const delaySeconds = i * 3600; 
+            const scheduleTimeSeconds = (Date.now() / 1000) + delaySeconds;
+
+            const taskPayload = {
+                action: 'EXECUTE_WINDOW',
+                planId: planId,
+                windowId: windowId,
+                userType: userType,
+                date: date
+            };
+
+            const taskRequest = {
+                parent: parentQueue,
+                task: {
+                    httpRequest: {
+                        httpMethod: 'POST',
+                        url: orchestratorUrl,
+                        headers: { 'Content-Type': 'application/json' },
+                        body: Buffer.from(JSON.stringify(taskPayload)).toString('base64'),
+                        oidcToken: {
+                            serviceAccountEmail: process.env.GCP_SERVICE_ACCOUNT_EMAIL
+                        }
+                    },
+                    scheduleTime: {
+                        seconds: scheduleTimeSeconds
+                    }
+                }
+            };
+            
+            tasksToCreate.push(taskRequest);
         }
     }
 
+    // Commit Firestore Writes First (Essential so tasks don't fail looking for data)
     await batchWriter.commit();
-    logger.log('SUCCESS', `[Orchestrator Plan] Plan Saved: ${planId} with ${writeCount} windows containing ${targets.length} users.`);
+    logger.log('SUCCESS', `[Orchestrator Plan] Plan ${planId} saved to Firestore.`);
+
+    // Dispatch Cloud Tasks
+    logger.log('INFO', `[Orchestrator Plan] Scheduling ${tasksToCreate.length} tasks for ${userType}...`);
+    
+    // Create tasks in parallel (with limit if needed, but usually fine for <50 tasks)
+    const scheduleResults = await Promise.allSettled(tasksToCreate.map(req => cloudTasksClient.createTask(req)));
+    
+    const failedCount = scheduleResults.filter(r => r.status === 'rejected').length;
+    if (failedCount > 0) {
+        logger.log('ERROR', `[Orchestrator Plan] Failed to schedule ${failedCount} tasks.`);
+        // Note: We don't rollback Firestore here; manual retry or idempotency handles it.
+    }
 
     return {
         planId: planId,
         totalUsers: targets.length,
-        windowCount: windowIds.length,
-        windowIds: windowIds
+        windowsCreated: tasksToCreate.length,
+        scheduledCount: tasksToCreate.length - failedCount,
+        message: `Plan created. ${tasksToCreate.length} windows scheduled over ${tasksToCreate.length} hours.`
     };
 }
 
 /**
  * LOGIC: Execute a specific window
- * 1. Reads the user list from Firestore.
- * 2. Calls dispatchUpdates to send them to the Task Engine.
+ * Triggered by Cloud Tasks when the schedule time arrives.
  */
 async function executeUpdateWindow(planId, windowId, userType, config, deps) {
     const { logger, db } = deps;
 
-    // 1. Fetch the window from Firestore
+    // 1. Fetch window from Firestore
     const windowRef = db.collection('system_update_plans').doc(planId).collection('windows').doc(String(windowId));
     const windowDoc = await windowRef.get();
 
@@ -156,7 +203,7 @@ async function executeUpdateWindow(planId, windowId, userType, config, deps) {
 
     const data = windowDoc.data();
     
-    // Idempotency check: prevent re-running a completed window
+    // Idempotency: Don't run if already done
     if (data.status === 'completed') {
         logger.log('WARN', `[Orchestrator Execute] Window ${windowId} already completed. Skipping.`);
         return { dispatchedCount: 0, status: 'already_completed' };
@@ -165,13 +212,12 @@ async function executeUpdateWindow(planId, windowId, userType, config, deps) {
     const targets = data.users;
     logger.log('INFO', `[Orchestrator Execute] Window ${windowId}: Dispatching ${targets.length} users.`);
 
-    // 2. Dispatch using existing helper
-    // The helper handles batching for Pub/Sub and logging.
+    // 2. Dispatch
     if (targets && targets.length > 0) {
         await dispatchUpdates(targets, userType, config.updateConfig, deps);
     }
 
-    // 3. Mark window as complete
+    // 3. Mark Complete
     await windowRef.update({ 
         status: 'completed',
         executedAt: FieldValue.serverTimestamp()
@@ -180,131 +226,52 @@ async function executeUpdateWindow(planId, windowId, userType, config, deps) {
     return { dispatchedCount: targets.length, status: 'success' };
 }
 
-/** Stage 1: Main discovery orchestrator pipe */
+// --- Legacy / Helper Wrappers (Preserved for compatibility) ---
+
 async function runDiscoveryOrchestrator(config, deps) {
     const { logger, firestoreUtils } = deps;
     logger.log('INFO', '🚀 Discovery Orchestrator triggered...');
     await firestoreUtils.resetProxyLocks(deps, config);
-    
-    // Check if normal user discovery is enabled
-    if (isUserTypeEnabled('normal', config.enabledUserTypes)) {
-        await runDiscovery('normal', config.discoveryConfig.normal, config, deps);
-    } else {
-        logger.log('INFO', '[Orchestrator] Normal user discovery is disabled. Skipping.');
-    }
-    
-    // Check if speculator discovery is enabled
-    if (isUserTypeEnabled('speculator', config.enabledUserTypes)) {
-        await runDiscovery('speculator', config.discoveryConfig.speculator, config, deps);
-    } else {
-        logger.log('INFO', '[Orchestrator] Speculator discovery is disabled. Skipping.');
-    }
+    if (isUserTypeEnabled('normal', config.enabledUserTypes)) await runDiscovery('normal', config.discoveryConfig.normal, config, deps);
+    if (isUserTypeEnabled('speculator', config.enabledUserTypes)) await runDiscovery('speculator', config.discoveryConfig.speculator, config, deps);
 }
 
-/** Stage 2: Main update orchestrator pipe */
 async function runUpdateOrchestrator(config, deps) {
     const { logger, firestoreUtils } = deps;
     logger.log('INFO', '🚀 Update Orchestrator triggered...');
     await firestoreUtils.resetProxyLocks(deps, config);
-    
     const enabledTypes = config.enabledUserTypes || [];
-    const enabledTypesStr = Array.isArray(enabledTypes) ? enabledTypes.join(', ') : String(enabledTypes);
-    logger.log('INFO', `[Orchestrator] Configuration loaded. Enabled user types: [${enabledTypesStr || 'none'}]`);
-    logger.log('INFO', `[Orchestrator] Config object keys: ${Object.keys(config).join(', ')}`);
-    logger.log('INFO', `[Orchestrator] enabledUserTypes type: ${typeof config.enabledUserTypes}, value: ${JSON.stringify(config.enabledUserTypes)}`);
-    
-    // 1. Normal Users
-    if (isUserTypeEnabled('normal', enabledTypes)) {
-        await runUpdates('normal', config.updateConfig, config, deps);
-    } else {
-        logger.log('INFO', '[Orchestrator] Normal user updates are disabled. Skipping.');
-    }
-    
-    // 2. Speculators
-    if (isUserTypeEnabled('speculator', enabledTypes)) {
-        await runUpdates('speculator', config.updateConfig, config, deps);
-    } else {
-        logger.log('INFO', '[Orchestrator] Speculator updates are disabled. Skipping.');
-    }
 
-    // 3. Popular Investors
+    if (isUserTypeEnabled('normal', enabledTypes)) await runUpdates('normal', config.updateConfig, config, deps);
+    if (isUserTypeEnabled('speculator', enabledTypes)) await runUpdates('speculator', config.updateConfig, config, deps);
     if (isUserTypeEnabled('popular_investor', enabledTypes)) {
-        try {
-            const piConfig = { 
-                ...config.updateConfig, 
-                popularInvestorRankingsCollection: config.updateConfig.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings'
-            };
-            await runUpdates('popular_investor', piConfig, config, deps);
-        } catch (e) {
-            logger.log('ERROR', '[Orchestrator] Failed to run Popular Investor updates.', e);
-        }
-    } else {
-        logger.log('INFO', '[Orchestrator] Popular Investor updates are disabled. Skipping.');
+        const piConfig = { ...config.updateConfig, popularInvestorRankingsCollection: config.updateConfig.popularInvestorRankingsCollection || 'popular_investor_rankings' };
+        await runUpdates('popular_investor', piConfig, config, deps);
     }
-
-    // 4. Signed-In Users
     if (isUserTypeEnabled('signed_in_user', enabledTypes)) {
-        try {
-            const signedInConfig = { 
-                ...config.updateConfig, 
-                signedInUsersCollection: config.updateConfig.signedInUsersCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users'
-            };
-            await runUpdates('signed_in_user', signedInConfig, config, deps);
-        } catch (e) {
-            logger.log('ERROR', '[Orchestrator] Failed to run Signed-In User updates.', e);
-        }
-    } else {
-        logger.log('INFO', '[Orchestrator] Signed-In User updates are disabled. Skipping.');
+        const signedInConfig = { ...config.updateConfig, signedInUsersCollection: config.updateConfig.signedInUsersCollection || 'signed_in_users' };
+        await runUpdates('signed_in_user', signedInConfig, config, deps);
     }
 }
 
-/** Stage 3: Run discovery for a single user type */
 async function runDiscovery(userType, userConfig, globalConfig, deps) {
     const { logger } = deps;
-    logger.log('INFO', `[Orchestrator] Starting discovery for ${userType} users...`);
-
-    // Step 3.1: Check if discovery is needed
     const { needsDiscovery, blocksToFill } = await checkDiscoveryNeed(userType, userConfig, deps);
-    if (!needsDiscovery) { logger.log('INFO', `[Orchestrator] No discovery needed for ${userType}.`); return; }
-
-    // Step 3.2: Get discovery candidates
+    if (!needsDiscovery) return;
     const candidates = await getDiscoveryCandidates(userType, blocksToFill, userConfig, deps);
-
-    // Step 3.3: Dispatch discovery tasks
     await dispatchDiscovery(userType, candidates, userConfig, deps);
-    logger.log('SUCCESS', `[Orchestrator] Dispatched discovery tasks for ${userType}.`);
 }
 
-/** Stage 4: Run updates for a single user type */
 async function runUpdates(userType, updateConfig, globalConfig, deps) {
-    const { logger } = deps;
-    logger.log('INFO', `[Orchestrator] Collecting users for daily update (${userType})...`);
-
-    // Step 4.1: Compute thresholds
     const now = new Date();
     const startOfTodayUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
-    const DaysAgoUTC = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
-    const thresholds = { dateThreshold: startOfTodayUTC, gracePeriodThreshold: DaysAgoUTC };
-
-    // Step 4.2: Get update targets
+    const thresholds = { dateThreshold: startOfTodayUTC, gracePeriodThreshold: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000) };
     const targets = await getUpdateTargets(userType, thresholds, updateConfig, deps);
-
-    // Step 4.3: Dispatch update tasks
     await dispatchUpdates(targets, userType, updateConfig, deps);
-    logger.log('SUCCESS', `[Orchestrator] Dispatched update tasks for ${userType}.`);
 }
 
-/**
- * Helper function to check if a user type is enabled
- * @param {string} userType - The user type to check ('normal', 'speculator', 'popular_investor', 'signed_in_user')
- * @param {Array<string>} enabledTypes - Array of enabled user types from config
- * @returns {boolean} True if the user type is enabled
- */
 function isUserTypeEnabled(userType, enabledTypes) {
-    if (!enabledTypes || !Array.isArray(enabledTypes) || enabledTypes.length === 0) {
-        // If no enabled types specified, default to all enabled (backward compatibility)
-        return true;
-    }
+    if (!enabledTypes || !Array.isArray(enabledTypes) || enabledTypes.length === 0) return true;
     return enabledTypes.includes(userType);
 }