bulltrackers-module 1.0.775 → 1.0.776

package/functions/computation-system-v2/handlers/scheduler.js

@@ -1,7 +1,7 @@
 /**
- * @fileoverview Scheduler V3.1: Reconciler with Active Garbage Collection
- * * * 1. Reconcile: Iterates a time window (Past -> Future) to ensure valid tasks exist.
- * * 2. Purge: Scans the queue for "Orphans" (tasks for deleted computations) and deletes them.
+ * @fileoverview Scheduler V3.2: Smart Reconciliation & Tombstone Aware
+ * * * 1. Reconcile: Checks ENTIRE graph for stale hashes, triggers ROOTS if needed.
+ * * 2. Purge: Actively removes tasks that don't match the current deployment hash.
  * * 3. Watchdog: Recovers "Zombie" tasks (running but stuck).
  */
 
@@ -41,8 +41,10 @@ async function initialize() {
 
     console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
 }
+
 /**
- * REPLACEMENT for planComputations
+ * ENTRY POINT 1: The Planner
+ * Trigger: Cloud Scheduler -> "0 * * * *" (Hourly)
  */
 async function planComputations(req, res) {
     try {
@@ -60,6 +62,7 @@ async function planComputations(req, res) {
         console.log(`[Planner] Reconciling window: ${windowStart.toISOString()} to ${windowEnd.toISOString()}`);
 
         // Helper to find Roots for any given computation (Pass 1..N)
+        // If a deep node (Pass 3) is stale, we must run its Pass 1 ancestor to regenerate the data.
         const manifestMap = new Map(manifest.map(m => [m.name, m]));
         const getRoots = (entry, visited = new Set()) => {
             if (visited.has(entry.name)) return [];
@@ -106,10 +109,12 @@ async function planComputations(req, res) {
                 if (reason) {
                     // If entry is stale, we must schedule its ROOT(s) to trigger the chain
                     const roots = getRoots(entry);
-
+                    
                     roots.forEach(root => {
+                        // Unique Task Key: RootName + Date + Hash
+                        // Including Hash in key ensures we don't dedupe against a DIFFERENT version
                         const taskKey = `root-${toKebab(root.originalName)}-${dateStr}-${root.hash}`;
-
+                        
                         if (!tasksToSchedule.has(taskKey)) {
                             tasksToSchedule.set(taskKey, {
                                 computation: root.originalName,
@@ -126,24 +131,28 @@ async function planComputations(req, res) {
         })));
 
         // --- PHASE 2: GARBAGE COLLECTION ---
-        // (Keep your existing GC logic here)
         console.log('[Planner] Starting Garbage Collection...');
         const deletedCount = await cleanupOrphanedTasks();
 
         // --- PHASE 3: DISPATCH ---
         const taskList = Array.from(tasksToSchedule.values());
         let scheduledCount = 0;
+        let existsCount = 0;
+
         if (taskList.length > 0) {
             const results = await dispatchTasks(taskList);
             scheduledCount = results.filter(r => r.status === 'scheduled').length;
+            existsCount = results.filter(r => r.status === 'exists').length;
         }
 
-        console.log(`[Planner] Complete. Scheduled: ${scheduledCount}, Deleted Orphans: ${deletedCount}`);
-
+        console.log(`[Planner] Complete. Scheduled: ${scheduledCount} (Exists/Tombstoned: ${existsCount}), Deleted Orphans: ${deletedCount}`);
+        
         return res.status(200).json({
             status: 'success',
             window: `${PLANNING_LOOKBACK_DAYS}d back`,
             scheduled: scheduledCount,
+            exists: existsCount,
+            deleted: deletedCount,
             stats
         });
 
@@ -188,13 +197,14 @@ async function runWatchdog(req, res) {
         return res.status(500).json({ error: error.message });
     }
 }
+
 // =============================================================================
 // ACTIVE GARBAGE COLLECTION LOGIC
 // =============================================================================
 
 async function cleanupOrphanedTasks() {
     const parent = getQueuePath();
-
+    
     // Create a map of { kebabName: activeHash } for O(1) lookups
     const activeComputations = new Map(
         manifest.map(m => [toKebab(m.originalName), m.hash])
@@ -206,18 +216,16 @@ async function cleanupOrphanedTasks() {
     try {
         const tasksToDelete = [];
 
-        // Note: listTasksAsync handles pagination, but if you have thousands of tasks,
-        // you might eventually need to handle page tokens explicitly if the library version is old.
-        for await (const task of tasksClient.listTasksAsync({
-            parent,
+        // Increase pageSize to 1000 to minimize pagination calls/warnings
+        for await (const task of tasksClient.listTasksAsync({ 
+            parent, 
             responseView: 'BASIC',
-            pageSize: 1000 // Increase page size to capture more per request
+            pageSize: 1000 
         })) {
             const taskNameFull = task.name;
-            const taskNameShort = taskNameFull.split('/').pop();
+            const taskNameShort = taskNameFull.split('/').pop(); 
 
             // 1. Handle ROOT Tasks: root-{kebabName}-{date}-{hash}
-            // We capture the name AND the hash at the end
             const rootMatch = taskNameShort.match(/^root-(.+)-\d{4}-\d{2}-\d{2}-(.+)$/);
 
             if (rootMatch) {
@@ -234,10 +242,8 @@ async function cleanupOrphanedTasks() {
             }
 
             // 2. Handle RECOVERY Tasks: recovery-{kebabName}-{date}-{timestamp}
-            // We only delete these if the computation is completely gone.
-            // (Timestamps won't match a config hash, so we just check existence)
             const recoveryMatch = taskNameShort.match(/^recovery-(.+)-\d{4}-\d{2}-\d{2}-/);
-
+            
             if (recoveryMatch) {
                 const [_, kebabName] = recoveryMatch;
                 if (!activeComputations.has(kebabName)) {
@@ -257,7 +263,7 @@ async function cleanupOrphanedTasks() {
                 deletedCount++;
             } catch (e) {
                 // Ignore "NOT_FOUND" errors in case of race conditions
-                if (e.code !== 5) {
+                if (e.code !== 5) { 
                     console.warn(`[Planner] Failed to delete ${name}: ${e.message}`);
                 }
             }
@@ -302,10 +308,10 @@ async function dispatchTasks(tasks) {
 
     return Promise.all(tasks.map(t => limit(async () => {
         try {
-            const name = t.isRecovery
+            const name = t.isRecovery 
                 ? `recovery-${toKebab(t.computation)}-${t.targetDate}-${Date.now()}`
                 : `root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
-
+            
             const taskName = `${t.queuePath}/tasks/${name}`;
 
             const payload = {
@@ -331,8 +337,16 @@ async function dispatchTasks(tasks) {
 
             await tasksClient.createTask({ parent: t.queuePath, task });
             return { status: 'scheduled' };
+            
         } catch (e) {
-            if (e.code === 6 || e.code === 409) return { status: 'exists' };
+            // ALREADY_EXISTS (6) or CONFLICT (409)
+            // This happens if we try to recreate a task that was recently deleted (Tombstone)
+            // or if it genuinely already exists.
+            if (e.code === 6 || e.code === 409) {
+                console.warn(`[Planner] Task skipped (Already Exists/Tombstone): ${t.computation} @ ${t.targetDate}`);
+                return { status: 'exists' };
+            }
+            
             console.error(`[Planner] Failed task ${t.computation}: ${e.message}`);
             return { status: 'error' };
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.775",
+  "version": "1.0.776",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [