bulltrackers-module 1.0.759 → 1.0.760

package/functions/computation-system-v2/handlers/dispatcher.js

@@ -1,33 +1,24 @@
 /**
  * @fileoverview Computation Dispatcher
- * * RESPONSIBILITIES:
- * 1. Receive HTTP request (from Cloud Tasks or User)
- * 2. Delegate execution to the Orchestrator
- * 3. Translate Orchestrator STATUS (blocked, completed) into HTTP CODES (503, 200)
- * * Why is this file so short?
- * All validation logic (Schedule, Dependencies, Data) has moved to 
- * framework/core/RunAnalyzer.js to ensure consistency between 
- * the Scheduler and the Worker.
+ * ...
  */
 
-// REMOVED: const system = require('../index'); 
-// We will lazy-load this inside the handler to prevent circular dependency cycles.
+const crypto = require('crypto');
 
 exports.dispatcherHandler = async (req, res) => {
     const startTime = Date.now();
     
     try {
-        // LAZY LOAD: Require index.js here to ensure it is fully initialized
-        // This fixes the "Accessing non-existent property inside circular dependency" error
         const system = require('../index');
 
         const {
             computationName,
             targetDate,
-            source = 'scheduled',    // 'scheduled' | 'on-demand' | 'zombie-recovery'
-            entityIds,               // Optional: specific entities
+            source = 'scheduled',
+            entityIds,
             dryRun = false,
-            force = false            // Optional: run even if hash matches
+            force = false,
+            configHash 
         } = req.body || {};
         
         // 1. Basic Validation
@@ -38,7 +29,37 @@ exports.dispatcherHandler = async (req, res) => {
                 message: 'computationName is required'
             });
         }
-        
+
+        // [FIXED LOGIC HERE] --------------------------------------------------
+        // Stale Task Protection
+        if (configHash && !force) {
+            // FIX: Use getManifest() as system.manifest is not exposed directly.
+            const manifest = await system.getManifest();
+            
+            // Normalize name to match manifest keys (matches logic in core-api.js)
+            const normalizedName = computationName.toLowerCase().replace(/[^a-z0-9]/g, '');
+            const entry = manifest.find(c => c.name === normalizedName);
+            
+            if (entry) {
+                // 1. Re-calculate the hash of the CURRENTLY DEPLOYED code
+                const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
+                const currentHash = crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
+                
+                // 2. Compare
+                if (configHash !== currentHash) {
+                    console.warn(`[Dispatcher] ♻️ Skipped STALE task for ${computationName}. (Task Hash: ${configHash} != Current: ${currentHash})`);
+                    
+                    return res.status(200).json({
+                        status: 'skipped',
+                        reason: 'STALE_CONFIG',
+                        message: 'Task configuration (schedule/pass) is obsolete relative to current deployment.',
+                        hash: currentHash
+                    });
+                }
+            }
+        }
+        // ---------------------------------------------------------------------
+
         const date = targetDate || new Date().toISOString().split('T')[0];
         console.log(`[Dispatcher] Received ${source} request: ${computationName} for ${date}`);
 
@@ -48,16 +69,12 @@ exports.dispatcherHandler = async (req, res) => {
         }
         
         // 2. DELEGATE TO ORCHESTRATOR
-        // The Orchestrator calls RunAnalyzer internally to check:
-        // - Is it scheduled?
-        // - Are dependencies ready?
-        // - Is data available?
-        // - Has the code changed?
         const result = await system.runComputation({
             date,
             computation: computationName,
             entityIds,
-            dryRun
+            dryRun,
+            force // Pass force down to orchestrator if needed
         });
 
         const duration = Date.now() - startTime;
@@ -78,11 +95,6 @@ exports.dispatcherHandler = async (req, res) => {
         } 
         
         // 4. HANDLE NON-RUNNABLE STATES (Blocked / Impossible)
-        // NEW BEHAVIOUR:
-        // - We NEVER return 503 for logical states like "blocked" or "impossible".
-        // - Cloud Tasks retries are reserved for genuine execution failures (5xx from errors).
-        // - Scheduler + dependency cascade should avoid dispatching truly blocked tasks;
-        //   if we still see them here, we surface the status once and BIN the task.
         if (result.status === 'blocked' || result.status === 'impossible') {
             console.log(`[Dispatcher] ${computationName} ${result.status}: ${result.reason}`);
 
@@ -93,14 +105,13 @@ exports.dispatcherHandler = async (req, res) => {
             });
         }
 
-        // 5. Fallback for other statuses (e.g., 'not_scheduled')
+        // 5. Fallback for other statuses
         return res.status(200).json(result);
         
     } catch (error) {
         const duration = Date.now() - startTime;
         console.error(`[Dispatcher] Error after ${duration}ms:`, error);
         
-        // Handle "Not Found" specifically
         if (error.message && error.message.includes('Computation not found')) {
             return res.status(400).json({
                 status: 'error',
@@ -109,7 +120,6 @@ exports.dispatcherHandler = async (req, res) => {
             });
         }
 
-        // Return 500 to trigger Cloud Tasks retry for crash/network errors
         return res.status(500).json({
             status: 'error',
             reason: 'EXECUTION_FAILED',

package/functions/computation-system-v2/handlers/scheduler.js

@@ -1,12 +1,16 @@
 /**
- * @fileoverview Unified Computation Scheduler (Refactored for Rolling Window)
- * * Triggered every minute.
- * * RESPONSIBILITY: Schedule ROOT tasks (Pass 0) only.
- * * STRATEGY: Look ahead 1 hour. Dispatch tasks with 'scheduleTime'. 
- * * DEDUPLICATION: Handled by Cloud Tasks 'name' property.
+ * @fileoverview Scheduler V2: Planner & Watchdog
+ * * 1. planComputations: Runs infrequently (e.g. Hourly/Daily).
+ * - Loads Manifest.
+ * - Forecasts all Root executions for the next 24-48h.
+ * - Enqueues Cloud Tasks with `scheduleTime` and `configHash`.
+ * * 2. runWatchdog: Runs frequently (e.g. every 15 mins).
+ * - Detects Zombies (stuck running tasks).
+ * - Re-queues them immediately.
  */
 
 const { CloudTasksClient } = require('@google-cloud/tasks');
+const crypto = require('crypto');
 const pLimit = require('p-limit');
 const { ManifestBuilder } = require('../framework');
 const { StorageManager } = require('../framework/storage/StorageManager');
@@ -14,188 +18,226 @@ const config = require('../config/bulltrackers.config');
 
 const CLOUD_TASKS_CONCURRENCY = 10;
 const ZOMBIE_THRESHOLD_MINUTES = 15;
+const PLANNING_WINDOW_HOURS = 24; // Look ahead window
 
+// Cache singleton instances
 let manifest = null;
 let tasksClient = null;
 let storageManager = null;
 
 async function initialize() {
     if (manifest) return;
+    console.log('[Scheduler] Initializing services...');
     
-    console.log('[Scheduler] Initializing...');
-    
-    // Core Services
-    // We pass a no-op logger to ManifestBuilder to keep logs clean during frequent scheduling
+    // We pass a no-op logger to prevent noise during frequent checks
     const builder = new ManifestBuilder(config, { log: () => {} });
     manifest = builder.build(config.computations || []);
     
-    // Infrastructure
     tasksClient = new CloudTasksClient();
     storageManager = new StorageManager(config, console);
     
-    console.log(`[Scheduler] Initialized with ${manifest.length} computations`);
+    console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
 }
 
-async function schedulerHandler(req, res) {
+/**
+ * ENTRY POINT 1: The Planner
+ * Trigger: Cloud Scheduler -> "0 * * * *" (Every Hour)
+ * Goals: Ensure all future tasks for the next 24h are in the queue.
+ */
+async function planComputations(req, res) {
     const startTime = Date.now();
-    
     try {
         await initialize();
-        
-        const now = new Date(); // Exact current time
-        const targetDate = now.toISOString().split('T')[0];
-
-        // 1. ROLLING WINDOW SCHEDULE
-        // Strategy: Look ahead 60 minutes. 
-        // If a task is due in this window, we dispatch it to Cloud Tasks with a 'scheduleTime'.
-        // Cloud Tasks deduplication (via task name) ensures we don't schedule it twice.
-        const windowEnd = new Date(now.getTime() + 60 * 60 * 1000);
-        
-        const dueComputations = findDueComputations(now, windowEnd);
-        
-        if (dueComputations.length > 0) {
-            console.log(`[Scheduler] Found ${dueComputations.length} Pass 0 tasks due between ${formatTime(now)} and ${formatTime(windowEnd)}`);
-        }
 
-        // 2. ZOMBIE DETECTION (Preserved from v1)
-        // Checks for tasks that started >15 mins ago but have no result and no recent heartbeat
-        let zombies = [];
-        try {
-            zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
+        const now = new Date();
+        const windowEnd = new Date(now.getTime() + PLANNING_WINDOW_HOURS * 60 * 60 * 1000);
+
+        console.log(`[Planner] Planning window: ${now.toISOString()} to ${windowEnd.toISOString()}`);
+
+        const tasksToSchedule = [];
+
+        // 1. Walk the Manifest
+        for (const entry of manifest) {
+            // FILTER: Only Roots (Pass 0)
+            // Resilience: If code changes and a comp becomes Pass 1, it won't be scheduled here.
+            if (entry.pass !== 0) continue;
+
+            // Calculate Occurrences
+            const occurrences = getOccurrencesInWindow(entry.schedule, now, windowEnd);
             
-            if (zombies.length > 0) {
-                const zombieDetails = zombies.map(z => `${z.name} [${z.date}]`).join(', ');
-                console.log(`[Scheduler] DETECTED ${zombies.length} ZOMBIES: ${zombieDetails}`);
-
-                // Claim zombies in DB to prevent re-dispatching in the next minute's run
-                await Promise.all(zombies.map(z => 
-                    storageManager.claimZombie(z.checkpointId) 
-                ));
+            // Generate Tasks for each occurrence
+            for (const dateObj of occurrences) {
+                // Resilience: Generate a hash of the critical scheduling config.
+                // If schedule OR pass changes, this hash changes, creating a new Task ID.
+                const configHash = generateConfigHash(entry);
+                const targetDateStr = dateObj.toISOString().split('T')[0];
+
+                tasksToSchedule.push({
+                    computation: entry.originalName,
+                    targetDate: targetDateStr,
+                    runAtSeconds: dateObj.getTime() / 1000,
+                    configHash: configHash,
+                    queuePath: getQueuePath(entry)
+                });
             }
-        } catch (e) {
-            console.error(`[Scheduler] Zombie check failed: ${e.message}`);
         }
 
-        // 3. EXIT IF NOTHING TO DO
-        if (dueComputations.length === 0 && zombies.length === 0) {
-            return res.status(200).json({ status: 'ok', message: 'Nothing due' });
+        if (tasksToSchedule.length === 0) {
+            return res.status(200).send('No root computations due in planning window.');
         }
 
-        // 4. PREPARE PAYLOADS
-        // Map zombies to the same structure as scheduled tasks
-        const zombieEntries = zombies.map(z => {
-            const originalEntry = manifest.find(m => m.name === z.name);
-            if (!originalEntry) return null;
-            
+        // 2. Dispatch to Cloud Tasks (Idempotent)
+        const results = await dispatchPlannedTasks(tasksToSchedule);
+
+        const created = results.filter(r => r.status === 'scheduled').length;
+        const exists = results.filter(r => r.status === 'exists').length;
+
+        console.log(`[Planner] Window Processed. Created: ${created}, Already Existed: ${exists}, Errors: ${results.length - created - exists}`);
+
+        return res.status(200).json({
+            status: 'ok',
+            window: `${PLANNING_WINDOW_HOURS}h`,
+            created,
+            exists,
+            details: results
+        });
+
+    } catch (error) {
+        console.error('[Planner] Fatal Error:', error);
+        return res.status(500).json({ error: error.message });
+    }
+}
+
+/**
+ * ENTRY POINT 2: The Watchdog
+ * Trigger: Cloud Scheduler -> "*\/15 * * * *" (Every 15 mins)
+ * Goals: Find stuck tasks and re-queue them.
+ */
+async function runWatchdog(req, res) {
+    try {
+        await initialize();
+        
+        // 1. Find Zombies
+        const zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
+        
+        if (zombies.length === 0) {
+            return res.status(200).send('No zombies detected.');
+        }
+
+        console.log(`[Watchdog] 🧟 Found ${zombies.length} zombies. Initiating recovery...`);
+
+        // 2. Claim & Recover
+        // We claim them first so the next watchdog doesn't grab them while we are dispatching
+        await Promise.all(zombies.map(z => storageManager.claimZombie(z.checkpointId)));
+
+        const recoveryTasks = zombies.map(z => {
+            const entry = manifest.find(m => m.name === z.name);
+            if (!entry) {
+                console.error(`[Watchdog] Computation ${z.name} no longer exists in manifest. Cannot recover.`);
+                return null;
+            }
             return {
-                ...originalEntry,
+                computation: entry.originalName,
+                targetDate: z.date,
                 isRecovery: true,
-                originalDate: z.date,
                 recoveryId: z.checkpointId,
-                runAt: 0 // Run immediately
+                queuePath: getQueuePath(entry)
             };
         }).filter(Boolean);
 
-        const allTasks = [...dueComputations, ...zombieEntries];
+        const results = await dispatchRecoveryTasks(recoveryTasks);
 
-        const results = await dispatchComputations(allTasks, targetDate);
-        
-        const duration = Date.now() - startTime;
-        
         return res.status(200).json({
-            status: 'ok',
-            dispatched: results.filter(r => r.status === 'dispatched').length,
-            duplicates: results.filter(r => r.status === 'skipped').length,
-            errors: results.filter(r => r.status === 'error').length,
-            duration,
-            results
+            status: 'recovered',
+            count: results.length,
+            details: results
         });
-        
+
     } catch (error) {
-        console.error('[Scheduler] Error:', error);
-        return res.status(500).json({ status: 'error', message: error.message });
+        console.error('[Watchdog] Error:', error);
+        return res.status(500).json({ error: error.message });
     }
 }
 
+// =============================================================================
+// HELPER FUNCTIONS
+// =============================================================================
+
 /**
- * Identify Pass 0 (Root) computations due within the time window.
+ * Calculates all execution times for a schedule within a start/end window.
+ * Returns Array<Date>
  */
-function findDueComputations(now, windowEnd) {
-    const due = [];
+function getOccurrencesInWindow(schedule, start, end) {
+    const times = [];
+    const [h, m] = (schedule.time || '02:00').split(':').map(Number);
     
-    for (const entry of manifest) {
-        // FILTER: Only Roots / Pass 0
-        // Any computation with dependencies is handled by the Orchestrator/Cascade system, not the Scheduler.
-        if (Array.isArray(entry.dependencies) && entry.dependencies.length > 0) {
-            continue;
-        }
+    // Clone start date to iterate
+    let current = new Date(start);
+    current.setUTCHours(h, m, 0, 0);
 
-        const nextRun = getNextExecutionTime(entry.schedule, now);
+    // If current is before start (e.g. window starts at 10:00, schedule is 02:00), move to tomorrow
+    if (current < start) {
+        current.setDate(current.getDate() + 1);
+    }
+
+    while (current <= end) {
+        let match = true;
+        
+        // Weekly Check
+        if (schedule.frequency === 'weekly' && current.getUTCDay() !== (schedule.dayOfWeek ?? 0)) {
+            match = false;
+        }
         
-        // CHECK: Is the calculated run time strictly within our window?
-        // Note: nextRun might be null if it doesn't run today (e.g. wrong day of week)
-        if (nextRun && nextRun >= now && nextRun <= windowEnd) {
-            due.push({
-                ...entry,
-                runAt: nextRun.getTime() / 1000 // Convert to Seconds for Cloud Tasks
-            });
+        // Monthly Check
+        if (schedule.frequency === 'monthly' && current.getUTCDate() !== (schedule.dayOfMonth ?? 1)) {
+            match = false;
+        }
+
+        if (match) {
+            times.push(new Date(current));
         }
+
+        // Advance 1 day
+        current.setDate(current.getDate() + 1);
     }
-    return due;
+    
+    return times;
 }
 
 /**
- * Calculate the specific execution Date object for a schedule relative to 'now'.
+ * Generates a short hash of the Scheduling Config + Pass.
+ * If this changes, we want a new Task ID to enforce the new schedule.
  */
-function getNextExecutionTime(schedule, now) {
-    // Simple implementation for Daily/Hourly schedules
-    // Format expected: "HH:mm" (24-hour)
-    const [h, m] = (schedule.time || '02:00').split(':').map(Number);
-    const target = new Date(now);
-    target.setUTCHours(h, m, 0, 0);
-    
-    // Day of Week check (for Weekly frequency)
-    // 0 = Sunday, 1 = Monday, etc.
-    if (schedule.frequency === 'weekly' && target.getUTCDay() !== (schedule.dayOfWeek ?? 0)) {
-        return null; 
-    }
-    
-    // Day of Month check (for Monthly frequency)
-    if (schedule.frequency === 'monthly' && target.getUTCDate() !== (schedule.dayOfMonth ?? 1)) {
-        return null;
-    }
+function generateConfigHash(entry) {
+    const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
+    return crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
+}
 
-    // Note: If 'target' is in the past (e.g. now is 03:00, schedule is 02:00),
-    // it will be filtered out by the window check (target >= now).
-    // We don't need to calculate "tomorrow's" run because the scheduler runs every minute;
-    // eventually "tomorrow" becomes "today".
-    
-    return target;
+function getQueuePath(entry) {
+    const { projectId, location, queueName } = config.cloudTasks;
+    return tasksClient.queuePath(projectId, location, queueName);
 }
 
-async function dispatchComputations(computations, defaultDate) {
+/**
+ * Dispatches Planned Root Tasks
+ * Uses deterministic naming for deduplication.
+ */
+async function dispatchPlannedTasks(tasks) {
     const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
-    const { projectId, location, queueName, dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
-    const queuePath = tasksClient.queuePath(projectId, location, queueName);
-    
-    const tasks = computations.map(entry => limit(async () => {
+    const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
+
+    return Promise.all(tasks.map(t => limit(async () => {
         try {
-            const taskDate = entry.isRecovery ? entry.originalDate : defaultDate;
-            const taskSource = entry.isRecovery ? 'zombie-recovery' : 'scheduled';
+            // Task Name: root-{name}-{date}-{configHash}
+            // If developer changes schedule -> hash changes -> new task created.
+            // If developer changes code but not schedule -> hash same -> existing task preserved.
+            const taskName = `${t.queuePath}/tasks/root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
             
-            // NAMING STRATEGY FOR DEDUPLICATION
-            // 1. Scheduled: compName-YYYYMMDD 
-            //    Ensures a daily task is only ever queued ONCE per day, even if scheduler overlaps.
-            // 2. Recovery:  compName-recovery-ID-timestamp 
-            //    Unique every time because we explicitly want to retry recovery.
-            const taskNameSuffix = entry.isRecovery 
-                ? `recovery-${entry.recoveryId}-${Date.now()}`
-                : `${taskDate}`; 
-
-            const taskPayload = {
-                computationName: entry.originalName,
-                targetDate: taskDate,
-                source: taskSource
+            const payload = {
+                computationName: t.computation,
+                targetDate: t.targetDate,
+                source: 'scheduled',
+                configHash: t.configHash // Sent to dispatcher for potential validation
             };
 
             const task = {
@@ -203,45 +245,68 @@ async function dispatchComputations(computations, defaultDate) {
                     httpMethod: 'POST',
                     url: dispatcherUrl,
                     headers: { 'Content-Type': 'application/json' },
-                    body: Buffer.from(JSON.stringify(taskPayload)).toString('base64'),
+                    body: Buffer.from(JSON.stringify(payload)).toString('base64'),
                     oidcToken: { serviceAccountEmail }
                 },
-                // Cloud Tasks handles the "wait until X" logic via scheduleTime
-                scheduleTime: entry.runAt > 0 ? { seconds: entry.runAt } : undefined,
-                name: `${queuePath}/tasks/${entry.name}-${taskNameSuffix}`
+                scheduleTime: { seconds: t.runAtSeconds },
+                name: taskName
             };
+
+            await tasksClient.createTask({ parent: t.queuePath, task });
+            return { computation: t.computation, date: t.targetDate, status: 'scheduled' };
+
+        } catch (e) {
+            if (e.code === 6 || e.code === 409) {
+                return { computation: t.computation, date: t.targetDate, status: 'exists' };
+            }
+            console.error(`[Planner] Failed to schedule ${t.computation}:`, e.message);
+            return { computation: t.computation, status: 'error', error: e.message };
+        }
+    })));
+}
+
+/**
+ * Dispatches Recovery Tasks (Zombies)
+ * Always creates unique task names to ensure retry.
+ */
+async function dispatchRecoveryTasks(tasks) {
+    const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
+    const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
+
+    return Promise.all(tasks.map(t => limit(async () => {
+        try {
+            // Unique ID for every recovery attempt
+            const taskName = `${t.queuePath}/tasks/recovery-${toKebab(t.computation)}-${t.recoveryId}-${Date.now()}`;
             
-            await tasksClient.createTask({ parent: queuePath, task });
-            
-            return { 
-                computation: entry.originalName, 
-                status: 'dispatched',
-                scheduledFor: entry.runAt > 0 ? new Date(entry.runAt * 1000).toISOString() : 'now'
+            const payload = {
+                computationName: t.computation,
+                targetDate: t.targetDate,
+                source: 'zombie-recovery'
             };
-            
-        } catch (error) {
-            // ALREADY_EXISTS (Code 6) or ABORTED/CONFLICT (Code 409)
-            // This is expected and desired behavior for the rolling window.
-            if (error.code === 6 || error.code === 409) { 
-                return { computation: entry.originalName, status: 'skipped', reason: 'duplicate' };
-            }
 
-            // Configuration Errors (Code 5: NOT_FOUND)
-            if (error.code === 5) {
-                console.error(`[Scheduler] 🚨 CONFIG ERROR: Queue '${queueName}' or SA '${serviceAccountEmail}' not found.`);
-                return { computation: entry.originalName, status: 'error', error: 'Config Error: Queue/SA not found' };
-            }
+            const task = {
+                httpRequest: {
+                    httpMethod: 'POST',
+                    url: dispatcherUrl,
+                    headers: { 'Content-Type': 'application/json' },
+                    body: Buffer.from(JSON.stringify(payload)).toString('base64'),
+                    oidcToken: { serviceAccountEmail }
+                },
+                // Run Immediately (no scheduleTime)
+                name: taskName
+            };
+
+            await tasksClient.createTask({ parent: t.queuePath, task });
+            return { computation: t.computation, status: 'recovered' };
 
-            console.error(`[Scheduler] Dispatch failed for ${entry.originalName}:`, error.message);
-            return { computation: entry.originalName, status: 'error', error: error.message };
+        } catch (e) {
+            return { computation: t.computation, status: 'error', error: e.message };
         }
-    }));
-    
-    return Promise.all(tasks);
+    })));
 }
 
-function formatTime(date) {
-    return date.toISOString().split('T')[1].substring(0, 5);
+function toKebab(str) {
+    return str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase().replace(/[^a-z0-9-]/g, '');
 }
 
-module.exports = { schedulerHandler, initialize };
+module.exports = { planComputations, runWatchdog };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.759",
+  "version": "1.0.760",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [