bulltrackers-module 1.0.769 → 1.0.771

package/functions/computation-system-v2/framework/storage/StateRepository.js

@@ -17,7 +17,7 @@ class StateRepository {
     constructor(config, logger = null) {
         this.config = config;
         this.logger = logger || console;
-        
+
         this.bigquery = new BigQuery({
             projectId: config.bigquery.projectId,
             location: config.bigquery.location
@@ -156,6 +156,8 @@ class StateRepository {
                 location: this.config.bigquery.location
             });
 
+            this._log('INFO', `getResult('${computationName}', '${dateStr}') table=${table} rows=${rows.length}`);
+
             if (rows.length === 0) {
                 this.resultCache.set(cacheKey, null);
                 return null;
@@ -213,8 +215,8 @@ class StateRepository {
 
             const [rows] = await this.bigquery.query({
                 query,
-                params: { 
-                    targetDate: dateStr, 
+                params: {
+                    targetDate: dateStr,
                     compName: computationName.toLowerCase(),
                     entityId: String(entityId)
                 },
@@ -243,11 +245,11 @@ class StateRepository {
      */
     async getBatchEntityResults(dateStr, computationName, entityIds) {
         if (!entityIds || entityIds.length === 0) return {};
-        
+
         const cacheKeyPrefix = `${dateStr}:${computationName.toLowerCase()}`;
         const results = {};
         const uncachedIds = [];
-        
+
         // Check cache first
         for (const entityId of entityIds) {
             const key = `${cacheKeyPrefix}:${entityId}`;
@@ -257,9 +259,9 @@ class StateRepository {
                 uncachedIds.push(entityId);
             }
         }
-        
+
         if (uncachedIds.length === 0) return results;
-        
+
         // Fetch uncached in batch
         try {
             const table = this.config.resultStore?.table || 'computation_results';
@@ -275,8 +277,8 @@ class StateRepository {
 
             const [rows] = await this.bigquery.query({
                 query,
-                params: { 
-                    targetDate: dateStr, 
+                params: {
+                    targetDate: dateStr,
                     compName: computationName.toLowerCase(),
                     entityIds: uncachedIds.map(String)
                 },
@@ -290,11 +292,11 @@ class StateRepository {
                 if (typeof data === 'string') {
                     try { data = JSON.parse(data); } catch (e) { /* keep */ }
                 }
-                
+
                 results[entityId] = data;
                 this.resultCache.set(`${cacheKeyPrefix}:${entityId}`, data);
             }
-            
+
             return results;
         } catch (e) {
             this._log('ERROR', `Batch fetch failed: ${e.message}`);

package/functions/computation-system-v2/handlers/scheduler.js

@@ -1,106 +1,130 @@
 /**
- * @fileoverview Scheduler V2: Planner & Watchdog
- * * 1. planComputations: Runs infrequently (e.g. Hourly/Daily).
- * - Loads Manifest.
- * - Forecasts all Root executions for the next 24-48h.
- * - Enqueues Cloud Tasks with `scheduleTime` and `configHash`.
- * * 2. runWatchdog: Runs frequently (e.g. every 15 mins).
- * - Detects Zombies (stuck running tasks).
- * - Re-queues them immediately.
+ * @fileoverview Scheduler V3.1: Reconciler with Active Garbage Collection
+ * * * 1. Reconcile: Iterates a time window (Past -> Future) to ensure valid tasks exist.
+ * * 2. Purge: Scans the queue for "Orphans" (tasks for deleted computations) and deletes them.
+ * * 3. Watchdog: Recovers "Zombie" tasks (running but stuck).
  */
 
 const { CloudTasksClient } = require('@google-cloud/tasks');
 const crypto = require('crypto');
 const pLimit = require('p-limit');
+
+// Framework
 const { ManifestBuilder } = require('../framework');
 const { StorageManager } = require('../framework/storage/StorageManager');
+const { StateRepository } = require('../framework/storage/StateRepository');
 const config = require('../config/bulltrackers.config');
 
-const CLOUD_TASKS_CONCURRENCY = 10;
+// Config
+const CLOUD_TASKS_CONCURRENCY = 20;
+const PLANNING_LOOKBACK_DAYS = 7;    // Look back to ensure recent history is correct
+const PLANNING_LOOKAHEAD_HOURS = 24; // Schedule future tasks
 const ZOMBIE_THRESHOLD_MINUTES = 15;
-const PLANNING_WINDOW_HOURS = 24; // Look ahead window
 
 // Cache singleton instances
 let manifest = null;
 let tasksClient = null;
 let storageManager = null;
+let stateRepository = null;
 
 async function initialize() {
     if (manifest) return;
     console.log('[Scheduler] Initializing services...');
-    
-    // We pass a no-op logger to prevent noise during frequent checks
-    const builder = new ManifestBuilder(config, { log: () => {} });
+
+    // Use a no-op logger for manifest builder to reduce noise
+    const builder = new ManifestBuilder(config, { log: () => { } });
     manifest = builder.build(config.computations || []);
-    
+
     tasksClient = new CloudTasksClient();
     storageManager = new StorageManager(config, console);
-    
+    stateRepository = new StateRepository(config, console);
+
     console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
 }
 
 /**
- * ENTRY POINT 1: The Planner
+ * ENTRY POINT 1: The Reconciler & Garbage Collector
  * Trigger: Cloud Scheduler -> "0 * * * *" (Every Hour)
- * Goals: Ensure all future tasks for the next 24h are in the queue.
  */
 async function planComputations(req, res) {
-    const startTime = Date.now();
     try {
         await initialize();
 
+        // --- PHASE 1: RECONCILIATION (Ensure valid tasks exist) ---
         const now = new Date();
-        const windowEnd = new Date(now.getTime() + PLANNING_WINDOW_HOURS * 60 * 60 * 1000);
+        const windowStart = new Date(now);
+        windowStart.setDate(now.getDate() - PLANNING_LOOKBACK_DAYS);
+        windowStart.setHours(0, 0, 0, 0);
 
-        console.log(`[Planner] Planning window: ${now.toISOString()} to ${windowEnd.toISOString()}`);
+        const windowEnd = new Date(now);
+        windowEnd.setTime(now.getTime() + (PLANNING_LOOKAHEAD_HOURS * 60 * 60 * 1000));
 
-        const tasksToSchedule = [];
+        console.log(`[Planner] Reconciling window: ${windowStart.toISOString()} to ${windowEnd.toISOString()}`);
 
-        // 1. Walk the Manifest
-        for (const entry of manifest) {
-            // FILTER: Only Roots (Pass 1)
-            // Graph.js assigns roots (no dependencies) to Pass 1.
-            // Resilience: If code changes and a comp becomes Pass 2+, it won't be scheduled here.
-            if (entry.pass !== 1) continue;
-
-            // Calculate Occurrences
-            const occurrences = getOccurrencesInWindow(entry.schedule, now, windowEnd);
-            
-            // Generate Tasks for each occurrence
-            for (const dateObj of occurrences) {
-                // Resilience: Generate a hash of the critical scheduling config.
-                // If schedule OR pass changes, this hash changes, creating a new Task ID.
-                const configHash = generateConfigHash(entry);
-                const targetDateStr = dateObj.toISOString().split('T')[0];
-
-                tasksToSchedule.push({
-                    computation: entry.originalName,
-                    targetDate: targetDateStr,
-                    runAtSeconds: dateObj.getTime() / 1000,
-                    configHash: configHash,
-                    queuePath: getQueuePath(entry)
-                });
-            }
+        const tasksToSchedule = [];
+        const stats = { checked: 0, scheduled: 0, mismatched: 0, missing: 0 };
+
+        // Iterate dates in window
+        const targetDates = [];
+        let cursor = new Date(windowStart);
+        while (cursor <= windowEnd) {
+            targetDates.push(new Date(cursor));
+            cursor.setDate(cursor.getDate() + 1);
         }
 
-        if (tasksToSchedule.length === 0) {
-            return res.status(200).send('No root computations due in planning window.');
-        }
+        const dateLimit = pLimit(5);
+        await Promise.all(targetDates.map(dateObj => dateLimit(async () => {
+            const dateStr = dateObj.toISOString().split('T')[0];
+            const dailyStatus = await stateRepository.getDailyStatus(dateStr);
+
+            for (const entry of manifest) {
+                if (entry.pass !== 1) continue; // Only schedule Roots
+                if (!shouldRunOnDate(entry.schedule, dateObj)) continue;
+
+                stats.checked++;
+                const lastRun = dailyStatus.get(entry.name);
+                let reason = null;
+
+                if (!lastRun) {
+                    reason = 'MISSING_RUN';
+                    stats.missing++;
+                } else if (lastRun.hash !== entry.hash) {
+                    reason = 'HASH_MISMATCH';
+                    stats.mismatched++;
+                }
+
+                if (reason) {
+                    tasksToSchedule.push({
+                        computation: entry.originalName,
+                        targetDate: dateStr,
+                        runAtSeconds: getRunTimeSeconds(entry.schedule, dateObj),
+                        configHash: entry.hash,
+                        queuePath: getQueuePath(),
+                        reason
+                    });
+                }
+            }
+        })));
 
-        // 2. Dispatch to Cloud Tasks (Idempotent)
-        const results = await dispatchPlannedTasks(tasksToSchedule);
+        // --- PHASE 2: GARBAGE COLLECTION (Remove invalid tasks) ---
+        console.log('[Planner] Starting Garbage Collection...');
+        const deletedCount = await cleanupOrphanedTasks();
 
-        const created = results.filter(r => r.status === 'scheduled').length;
-        const exists = results.filter(r => r.status === 'exists').length;
+        // --- PHASE 3: DISPATCH ---
+        let scheduledCount = 0;
+        if (tasksToSchedule.length > 0) {
+            const results = await dispatchTasks(tasksToSchedule);
+            scheduledCount = results.filter(r => r.status === 'scheduled').length;
+        }
 
-        console.log(`[Planner] Window Processed. Created: ${created}, Already Existed: ${exists}, Errors: ${results.length - created - exists}`);
+        console.log(`[Planner] Complete. Scheduled: ${scheduledCount}, Deleted Orphans: ${deletedCount}`);
 
         return res.status(200).json({
-            status: 'ok',
-            window: `${PLANNING_WINDOW_HOURS}h`,
-            created,
-            exists,
-            details: results
+            status: 'success',
+            window: `${PLANNING_LOOKBACK_DAYS}d back, ${PLANNING_LOOKAHEAD_HOURS}h fwd`,
+            scheduled: scheduledCount,
+            deletedOrphans: deletedCount,
+            stats
         });
 
     } catch (error) {
@@ -112,57 +136,32 @@ async function planComputations(req, res) {
 /**
  * ENTRY POINT 2: The Watchdog
  * Trigger: Cloud Scheduler -> "*\/15 * * * *" (Every 15 mins)
- * Goals: Find stuck tasks and re-queue them.
  */
 async function runWatchdog(req, res) {
     try {
         await initialize();
-        
-        // 1. Find Zombies
         const zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
-        
-        // Filter out excessive attempts
-        const actionableZombies = [];
-        for (const z of zombies) {
-            if ((z.attempts || 0) >= 3) {
-                console.warn(`[Watchdog] Ignoring zombie ${z.name} (Checkpoint: ${z.checkpointId}) - Max attempts reached (${z.attempts})`);
-                continue;
-            }
-            actionableZombies.push(z);
-        }
-
-        if (actionableZombies.length === 0) {
-            return res.status(200).send('No recoverable zombies.');
-        }
+        const actionableZombies = zombies.filter(z => (z.attempts || 0) < 3);
 
-        console.log(`[Watchdog] 🧟 Found ${actionableZombies.length} zombies. Initiating recovery...`);
+        if (actionableZombies.length === 0) return res.status(200).send('No recoverable zombies.');
 
-        // 2. Claim & Recover
-        // We claim them first so the next watchdog doesn't grab them while we are dispatching
+        console.log(`[Watchdog] 🧟 Found ${actionableZombies.length} zombies. Recovering...`);
         await Promise.all(actionableZombies.map(z => storageManager.claimZombie(z.checkpointId)));
 
         const recoveryTasks = actionableZombies.map(z => {
             const entry = manifest.find(m => m.name === z.name);
-            if (!entry) {
-                console.error(`[Watchdog] Computation ${z.name} no longer exists in manifest. Cannot recover.`);
-                return null;
-            }
+            if (!entry) return null;
             return {
                 computation: entry.originalName,
                 targetDate: z.date,
                 isRecovery: true,
                 recoveryId: z.checkpointId,
-                queuePath: getQueuePath(entry)
+                queuePath: getQueuePath()
             };
         }).filter(Boolean);
 
-        const results = await dispatchRecoveryTasks(recoveryTasks);
-
-        return res.status(200).json({
-            status: 'recovered',
-            count: results.length,
-            details: results
-        });
+        const results = await dispatchTasks(recoveryTasks);
+        return res.status(200).json({ status: 'recovered', count: results.length });
 
     } catch (error) {
         console.error('[Watchdog] Error:', error);
@@ -171,131 +170,106 @@ async function runWatchdog(req, res) {
 }
 
 // =============================================================================
-// HELPER FUNCTIONS
+// ACTIVE GARBAGE COLLECTION LOGIC
 // =============================================================================
 
-/**
- * Calculates all execution times for a schedule within a start/end window.
- * Returns Array<Date>
- */
-function getOccurrencesInWindow(schedule, start, end) {
-    const times = [];
-    const [h, m] = (schedule.time || '02:00').split(':').map(Number);
-    
-    // Clone start date to iterate
-    let current = new Date(start);
-    current.setUTCHours(h, m, 0, 0);
-
-    // If current is before start (e.g. window starts at 10:00, schedule is 02:00), move to tomorrow
-    if (current < start) {
-        current.setDate(current.getDate() + 1);
-    }
+async function cleanupOrphanedTasks() {
+    const parent = getQueuePath();
+    const validKebabNames = new Set(manifest.map(m => toKebab(m.originalName)));
+    const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
+    let deletedCount = 0;
 
-    while (current <= end) {
-        let match = true;
-        
-        // Weekly Check
-        if (schedule.frequency === 'weekly' && current.getUTCDay() !== (schedule.dayOfWeek ?? 0)) {
-            match = false;
-        }
-        
-        // Monthly Check
-        if (schedule.frequency === 'monthly' && current.getUTCDate() !== (schedule.dayOfMonth ?? 1)) {
-            match = false;
-        }
+    try {
+        // Iterate over ALL tasks in the queue
+        // Note: listTasksAsync handles pagination automatically
+        const tasksToDelete = [];
+
+        for await (const task of tasksClient.listTasksAsync({ parent, responseView: 'BASIC' })) {
+            const taskNameFull = task.name;
+            const taskNameShort = taskNameFull.split('/').pop(); // e.g., root-my-comp-2023-01-01-abcdef
 
-        if (match) {
-            times.push(new Date(current));
+            // 1. Regex Match: Capture the computation name part
+            // Pattern: (root|recovery)-{kebabName}-{date}-{hash}
+            // Date is YYYY-MM-DD (10 chars)
+            // Hash is 8 chars (or more)
+            const match = taskNameShort.match(/^(?:root|recovery)-(.+)-\d{4}-\d{2}-\d{2}-/);
+
+            if (!match) continue; // Skip tasks that don't match our naming convention
+
+            const extractedKebabName = match[1];
+
+            // 2. Check Validity
+            if (!validKebabNames.has(extractedKebabName)) {
+                // ORPHAN DETECTED!
+                tasksToDelete.push(taskNameFull);
+            }
         }
 
-        // Advance 1 day
-        current.setDate(current.getDate() + 1);
+        if (tasksToDelete.length === 0) return 0;
+
+        console.log(`[Planner] 🗑️ Found ${tasksToDelete.length} orphaned tasks. Deleting...`);
+
+        // 3. Delete in parallel
+        await Promise.all(tasksToDelete.map(name => limit(async () => {
+            try {
+                await tasksClient.deleteTask({ name });
+                deletedCount++;
+            } catch (e) {
+                console.warn(`[Planner] Failed to delete orphan ${name}: ${e.message}`);
+            }
+        })));
+
+    } catch (e) {
+        console.error(`[Planner] GC Error: ${e.message}`);
     }
-    
-    return times;
+
+    return deletedCount;
 }
 
-/**
- * Generates a short hash of the Scheduling Config + Pass.
- * If this changes, we want a new Task ID to enforce the new schedule.
- */
-function generateConfigHash(entry) {
-    const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
-    return crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function shouldRunOnDate(schedule, dateObj) {
+    if (schedule.frequency === 'weekly' && dateObj.getUTCDay() !== (schedule.dayOfWeek ?? 0)) return false;
+    if (schedule.frequency === 'monthly' && dateObj.getUTCDate() !== (schedule.dayOfMonth ?? 1)) return false;
+    return true;
+}
+
+function getRunTimeSeconds(schedule, dateObj) {
+    const [h, m] = (schedule.time || '02:00').split(':').map(Number);
+    const runTime = new Date(dateObj);
+    runTime.setUTCHours(h, m, 0, 0);
+    return runTime.getTime() / 1000;
 }
 
-function getQueuePath(entry) {
+function getQueuePath() {
     const { projectId, location, queueName } = config.cloudTasks;
     return tasksClient.queuePath(projectId, location, queueName);
 }
 
-/**
- * Dispatches Planned Root Tasks
- * Uses deterministic naming for deduplication.
- */
-async function dispatchPlannedTasks(tasks) {
-    const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
-    const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
-
-    return Promise.all(tasks.map(t => limit(async () => {
-        try {
-            // Task Name: root-{name}-{date}-{configHash}
-            // If developer changes schedule -> hash changes -> new task created.
-            // If developer changes code but not schedule -> hash same -> existing task preserved.
-            const taskName = `${t.queuePath}/tasks/root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
-            
-            const payload = {
-                computationName: t.computation,
-                targetDate: t.targetDate,
-                source: 'scheduled',
-                configHash: t.configHash // Sent to dispatcher for potential validation
-            };
-
-            const task = {
-                httpRequest: {
-                    httpMethod: 'POST',
-                    url: dispatcherUrl,
-                    headers: { 'Content-Type': 'application/json' },
-                    body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: { 
-                        serviceAccountEmail,
-                        audience: dispatcherUrl // <--- FIXED: Must exactly match function URL
-                    }
-                },
-                scheduleTime: { seconds: t.runAtSeconds },
-                name: taskName
-            };
-
-            await tasksClient.createTask({ parent: t.queuePath, task });
-            return { computation: t.computation, date: t.targetDate, status: 'scheduled' };
-
-        } catch (e) {
-            if (e.code === 6 || e.code === 409) {
-                return { computation: t.computation, date: t.targetDate, status: 'exists' };
-            }
-            console.error(`[Planner] Failed to schedule ${t.computation}:`, e.message);
-            return { computation: t.computation, status: 'error', error: e.message };
-        }
-    })));
+function toKebab(str) {
+    return str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase().replace(/[^a-z0-9-]/g, '');
 }
 
-/**
- * Dispatches Recovery Tasks (Zombies)
- * Always creates unique task names to ensure retry.
- */
-async function dispatchRecoveryTasks(tasks) {
+async function dispatchTasks(tasks) {
     const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
     const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
 
     return Promise.all(tasks.map(t => limit(async () => {
         try {
-            // Unique ID for every recovery attempt
-            const taskName = `${t.queuePath}/tasks/recovery-${toKebab(t.computation)}-${t.recoveryId}-${Date.now()}`;
-            
+            const name = t.isRecovery
+                ? `recovery-${toKebab(t.computation)}-${t.targetDate}-${Date.now()}`
+                : `root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
+
+            const taskName = `${t.queuePath}/tasks/${name}`;
+
             const payload = {
                 computationName: t.computation,
                 targetDate: t.targetDate,
-                source: 'zombie-recovery'
+                source: t.isRecovery ? 'zombie-recovery' : 'scheduled',
+                configHash: t.configHash,
+                reason: t.reason
             };
 
             const task = {
@@ -304,26 +278,21 @@ async function dispatchRecoveryTasks(tasks) {
                     url: dispatcherUrl,
                     headers: { 'Content-Type': 'application/json' },
                     body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: { 
-                        serviceAccountEmail,
-                        audience: dispatcherUrl // <--- FIXED: Must exactly match function URL
-                    }
+                    oidcToken: { serviceAccountEmail, audience: dispatcherUrl }
                 },
-                // Run Immediately (no scheduleTime)
                 name: taskName
             };
 
-            await tasksClient.createTask({ parent: t.queuePath, task });
-            return { computation: t.computation, status: 'recovered' };
+            if (t.runAtSeconds) task.scheduleTime = { seconds: t.runAtSeconds };
 
+            await tasksClient.createTask({ parent: t.queuePath, task });
+            return { status: 'scheduled' };
         } catch (e) {
-            return { computation: t.computation, status: 'error', error: e.message };
+            if (e.code === 6 || e.code === 409) return { status: 'exists' };
+            console.error(`[Planner] Failed task ${t.computation}: ${e.message}`);
+            return { status: 'error' };
         }
     })));
 }
 
-function toKebab(str) {
-    return str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase().replace(/[^a-z0-9-]/g, '');
-}
-
 module.exports = { planComputations, runWatchdog };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.769",
+  "version": "1.0.771",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system-v2/computations/SignedInUserList.js

@@ -1,51 +0,0 @@
-const { Computation } = require('../framework');
-
-class DebugSignedInUsers extends Computation {
-    static getConfig() {
-        return {
-            name: 'DebugSignedInUsers',
-            type: 'per-entity',
-            category: 'debug',
-            
-            // CRITICAL: Tells the Orchestrator which partition to use
-            userType: 'SIGNED_IN_USER', 
-
-            requires: {
-                // DRIVER TABLE: This MUST be first.
-                // The DAG will only create tasks for Entity IDs found in this filtered query.
-                'portfolio_snapshots': {
-                    lookback: 0, // Just need today's existence
-                    mandatory: true,
-                    fields: ['user_id', 'date'],
-                    filter: { user_type: 'SIGNED_IN_USER' }
-                }
-            },
-            
-            storage: {
-                // Just log to console or a debug table for now
-                bigquery: false, 
-                firestore: { enabled: false } 
-            }
-        };
-    }
-
-    async process(context) {
-        const { entityId, data } = context;
-        
-        // If the Orchestrator works correctly, we should ONLY see Signed-In Users here.
-        // We verify by returning their ID.
-        
-        const row = data['portfolio_snapshots'];
-        
-        // Sanity Check: If row exists, we are good.
-        if (row) {
-             this.setResult(entityId, {
-                 status: 'Found',
-                 userType: 'SIGNED_IN_USER',
-                 checkedAt: new Date().toISOString()
-             });
-        }
-    }
-}
-
-module.exports = DebugSignedInUsers;