bulltrackers-module 1.0.769 → 1.0.770

package/functions/computation-system-v2/computations/RiskScoreIncrease.js

@@ -6,17 +6,17 @@ class RiskScoreIncrease extends Computation {
         return {
             name: 'RiskScoreIncrease',
             // V2 Migration: Switch to per-entity for parallel processing
-            type: 'per-entity', 
+            type: 'per-entity',
             category: 'alerts',
             isHistorical: true,
-            
+
             // V2 Migration: Define strict data requirements
             requires: {
-                'rankings': {
+                'pi_rankings': {
                     lookback: 1, // Fetches Today + Yesterday
                     mandatory: true,
                     // Optimization: Only fetch needed fields
-                    fields: ['user_id', 'rankings_data', 'date'],
+                    fields: ['pi_id', 'rankings_data', 'date'],
                     // Filter applied at data fetch level (replacing V1 userType logic)
                     filter: { user_type: 'POPULAR_INVESTOR' }
                 }
@@ -28,7 +28,7 @@ class RiskScoreIncrease extends Computation {
                 firestore: {
                     enabled: true,
                     // Ensures V1 alert system can find the document
-                    path: 'alerts/{date}/RiskScoreIncrease/{entityId}', 
+                    path: 'alerts/{date}/RiskScoreIncrease/{entityId}',
                     merge: true
                 }
             },
@@ -44,7 +44,7 @@ class RiskScoreIncrease extends Computation {
                 description: 'Alert when a Popular Investor\'s risk score increases',
                 messageTemplate: '{piUsername}\'s risk score increased by {change} points (from {previous} to {current})',
                 severity: 'high',
-                configKey: 'increasedRisk', 
+                configKey: 'increasedRisk',
                 isDynamic: true,
                 dynamicConfig: {
                     thresholds: [
@@ -72,9 +72,9 @@ class RiskScoreIncrease extends Computation {
                         }
                     ],
                     resultFields: {
-                        change: 'change',           
-                        newValue: 'currentRisk',    
-                        oldValue: 'previousRisk'    
+                        change: 'change',
+                        newValue: 'currentRisk',
+                        oldValue: 'previousRisk'
                     }
                 }
             }
@@ -87,15 +87,15 @@ class RiskScoreIncrease extends Computation {
         // 1. Data Access
         // In V2 with lookback: 1, this is an array of entries sorted by date
         const rankingHistory = data['rankings'] || [];
-        
+
         // Find today's and yesterday's entries specifically by date string
         const todayEntry = rankingHistory.find(d => d.date === date);
-        
+
         // Calculate yesterday's date string for lookup
         const yesterdayDate = new Date(date);
         yesterdayDate.setDate(yesterdayDate.getDate() - 1);
         const yesterdayStr = yesterdayDate.toISOString().split('T')[0];
-        
+
         const yesterdayEntry = rankingHistory.find(d => d.date === yesterdayStr);
 
         // 2. Business Logic (Using Rules)
@@ -112,7 +112,7 @@ class RiskScoreIncrease extends Computation {
         // If today's risk data is missing, we cannot evaluate
         if (currentRisk === null || currentRisk === undefined) {
             this.log('WARN', `No current risk data for ${entityId}`);
-            return; 
+            return;
         }
 
         // 4. Comparison Logic

package/functions/computation-system-v2/handlers/scheduler.js

@@ -1,106 +1,130 @@
 /**
- * @fileoverview Scheduler V2: Planner & Watchdog
- * * 1. planComputations: Runs infrequently (e.g. Hourly/Daily).
- * - Loads Manifest.
- * - Forecasts all Root executions for the next 24-48h.
- * - Enqueues Cloud Tasks with `scheduleTime` and `configHash`.
- * * 2. runWatchdog: Runs frequently (e.g. every 15 mins).
- * - Detects Zombies (stuck running tasks).
- * - Re-queues them immediately.
+ * @fileoverview Scheduler V3.1: Reconciler with Active Garbage Collection
+ * * * 1. Reconcile: Iterates a time window (Past -> Future) to ensure valid tasks exist.
+ * * 2. Purge: Scans the queue for "Orphans" (tasks for deleted computations) and deletes them.
+ * * 3. Watchdog: Recovers "Zombie" tasks (running but stuck).
  */
 
 const { CloudTasksClient } = require('@google-cloud/tasks');
 const crypto = require('crypto');
 const pLimit = require('p-limit');
+
+// Framework
 const { ManifestBuilder } = require('../framework');
 const { StorageManager } = require('../framework/storage/StorageManager');
+const { StateRepository } = require('../framework/storage/StateRepository');
 const config = require('../config/bulltrackers.config');
 
-const CLOUD_TASKS_CONCURRENCY = 10;
+// Config
+const CLOUD_TASKS_CONCURRENCY = 20;
+const PLANNING_LOOKBACK_DAYS = 7;    // Look back to ensure recent history is correct
+const PLANNING_LOOKAHEAD_HOURS = 24; // Schedule future tasks
 const ZOMBIE_THRESHOLD_MINUTES = 15;
-const PLANNING_WINDOW_HOURS = 24; // Look ahead window
 
 // Cache singleton instances
 let manifest = null;
 let tasksClient = null;
 let storageManager = null;
+let stateRepository = null;
 
 async function initialize() {
     if (manifest) return;
     console.log('[Scheduler] Initializing services...');
-    
-    // We pass a no-op logger to prevent noise during frequent checks
-    const builder = new ManifestBuilder(config, { log: () => {} });
+
+    // Use a no-op logger for manifest builder to reduce noise
+    const builder = new ManifestBuilder(config, { log: () => { } });
     manifest = builder.build(config.computations || []);
-    
+
     tasksClient = new CloudTasksClient();
     storageManager = new StorageManager(config, console);
-    
+    stateRepository = new StateRepository(config, console);
+
     console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
 }
 
 /**
- * ENTRY POINT 1: The Planner
+ * ENTRY POINT 1: The Reconciler & Garbage Collector
  * Trigger: Cloud Scheduler -> "0 * * * *" (Every Hour)
- * Goals: Ensure all future tasks for the next 24h are in the queue.
  */
 async function planComputations(req, res) {
-    const startTime = Date.now();
     try {
         await initialize();
 
+        // --- PHASE 1: RECONCILIATION (Ensure valid tasks exist) ---
         const now = new Date();
-        const windowEnd = new Date(now.getTime() + PLANNING_WINDOW_HOURS * 60 * 60 * 1000);
+        const windowStart = new Date(now);
+        windowStart.setDate(now.getDate() - PLANNING_LOOKBACK_DAYS);
+        windowStart.setHours(0, 0, 0, 0);
 
-        console.log(`[Planner] Planning window: ${now.toISOString()} to ${windowEnd.toISOString()}`);
+        const windowEnd = new Date(now);
+        windowEnd.setTime(now.getTime() + (PLANNING_LOOKAHEAD_HOURS * 60 * 60 * 1000));
 
-        const tasksToSchedule = [];
+        console.log(`[Planner] Reconciling window: ${windowStart.toISOString()} to ${windowEnd.toISOString()}`);
 
-        // 1. Walk the Manifest
-        for (const entry of manifest) {
-            // FILTER: Only Roots (Pass 1)
-            // Graph.js assigns roots (no dependencies) to Pass 1.
-            // Resilience: If code changes and a comp becomes Pass 2+, it won't be scheduled here.
-            if (entry.pass !== 1) continue;
-
-            // Calculate Occurrences
-            const occurrences = getOccurrencesInWindow(entry.schedule, now, windowEnd);
-            
-            // Generate Tasks for each occurrence
-            for (const dateObj of occurrences) {
-                // Resilience: Generate a hash of the critical scheduling config.
-                // If schedule OR pass changes, this hash changes, creating a new Task ID.
-                const configHash = generateConfigHash(entry);
-                const targetDateStr = dateObj.toISOString().split('T')[0];
-
-                tasksToSchedule.push({
-                    computation: entry.originalName,
-                    targetDate: targetDateStr,
-                    runAtSeconds: dateObj.getTime() / 1000,
-                    configHash: configHash,
-                    queuePath: getQueuePath(entry)
-                });
-            }
+        const tasksToSchedule = [];
+        const stats = { checked: 0, scheduled: 0, mismatched: 0, missing: 0 };
+
+        // Iterate dates in window
+        const targetDates = [];
+        let cursor = new Date(windowStart);
+        while (cursor <= windowEnd) {
+            targetDates.push(new Date(cursor));
+            cursor.setDate(cursor.getDate() + 1);
         }
 
-        if (tasksToSchedule.length === 0) {
-            return res.status(200).send('No root computations due in planning window.');
-        }
+        const dateLimit = pLimit(5);
+        await Promise.all(targetDates.map(dateObj => dateLimit(async () => {
+            const dateStr = dateObj.toISOString().split('T')[0];
+            const dailyStatus = await stateRepository.getDailyStatus(dateStr);
+
+            for (const entry of manifest) {
+                if (entry.pass !== 1) continue; // Only schedule Roots
+                if (!shouldRunOnDate(entry.schedule, dateObj)) continue;
+
+                stats.checked++;
+                const lastRun = dailyStatus.get(entry.name);
+                let reason = null;
+
+                if (!lastRun) {
+                    reason = 'MISSING_RUN';
+                    stats.missing++;
+                } else if (lastRun.hash !== entry.hash) {
+                    reason = 'HASH_MISMATCH';
+                    stats.mismatched++;
+                }
+
+                if (reason) {
+                    tasksToSchedule.push({
+                        computation: entry.originalName,
+                        targetDate: dateStr,
+                        runAtSeconds: getRunTimeSeconds(entry.schedule, dateObj),
+                        configHash: entry.hash,
+                        queuePath: getQueuePath(),
+                        reason
+                    });
+                }
+            }
+        })));
 
-        // 2. Dispatch to Cloud Tasks (Idempotent)
-        const results = await dispatchPlannedTasks(tasksToSchedule);
+        // --- PHASE 2: GARBAGE COLLECTION (Remove invalid tasks) ---
+        console.log('[Planner] Starting Garbage Collection...');
+        const deletedCount = await cleanupOrphanedTasks();
 
-        const created = results.filter(r => r.status === 'scheduled').length;
-        const exists = results.filter(r => r.status === 'exists').length;
+        // --- PHASE 3: DISPATCH ---
+        let scheduledCount = 0;
+        if (tasksToSchedule.length > 0) {
+            const results = await dispatchTasks(tasksToSchedule);
+            scheduledCount = results.filter(r => r.status === 'scheduled').length;
+        }
 
-        console.log(`[Planner] Window Processed. Created: ${created}, Already Existed: ${exists}, Errors: ${results.length - created - exists}`);
+        console.log(`[Planner] Complete. Scheduled: ${scheduledCount}, Deleted Orphans: ${deletedCount}`);
 
         return res.status(200).json({
-            status: 'ok',
-            window: `${PLANNING_WINDOW_HOURS}h`,
-            created,
-            exists,
-            details: results
+            status: 'success',
+            window: `${PLANNING_LOOKBACK_DAYS}d back, ${PLANNING_LOOKAHEAD_HOURS}h fwd`,
+            scheduled: scheduledCount,
+            deletedOrphans: deletedCount,
+            stats
         });
 
     } catch (error) {
@@ -112,57 +136,32 @@ async function planComputations(req, res) {
 /**
  * ENTRY POINT 2: The Watchdog
  * Trigger: Cloud Scheduler -> "*\/15 * * * *" (Every 15 mins)
- * Goals: Find stuck tasks and re-queue them.
  */
 async function runWatchdog(req, res) {
     try {
         await initialize();
-        
-        // 1. Find Zombies
         const zombies = await storageManager.findZombies(ZOMBIE_THRESHOLD_MINUTES);
-        
-        // Filter out excessive attempts
-        const actionableZombies = [];
-        for (const z of zombies) {
-            if ((z.attempts || 0) >= 3) {
-                console.warn(`[Watchdog] Ignoring zombie ${z.name} (Checkpoint: ${z.checkpointId}) - Max attempts reached (${z.attempts})`);
-                continue;
-            }
-            actionableZombies.push(z);
-        }
-
-        if (actionableZombies.length === 0) {
-            return res.status(200).send('No recoverable zombies.');
-        }
+        const actionableZombies = zombies.filter(z => (z.attempts || 0) < 3);
 
-        console.log(`[Watchdog] 🧟 Found ${actionableZombies.length} zombies. Initiating recovery...`);
+        if (actionableZombies.length === 0) return res.status(200).send('No recoverable zombies.');
 
-        // 2. Claim & Recover
-        // We claim them first so the next watchdog doesn't grab them while we are dispatching
+        console.log(`[Watchdog] 🧟 Found ${actionableZombies.length} zombies. Recovering...`);
         await Promise.all(actionableZombies.map(z => storageManager.claimZombie(z.checkpointId)));
 
         const recoveryTasks = actionableZombies.map(z => {
             const entry = manifest.find(m => m.name === z.name);
-            if (!entry) {
-                console.error(`[Watchdog] Computation ${z.name} no longer exists in manifest. Cannot recover.`);
-                return null;
-            }
+            if (!entry) return null;
             return {
                 computation: entry.originalName,
                 targetDate: z.date,
                 isRecovery: true,
                 recoveryId: z.checkpointId,
-                queuePath: getQueuePath(entry)
+                queuePath: getQueuePath()
             };
         }).filter(Boolean);
 
-        const results = await dispatchRecoveryTasks(recoveryTasks);
-
-        return res.status(200).json({
-            status: 'recovered',
-            count: results.length,
-            details: results
-        });
+        const results = await dispatchTasks(recoveryTasks);
+        return res.status(200).json({ status: 'recovered', count: results.length });
 
     } catch (error) {
         console.error('[Watchdog] Error:', error);
@@ -171,131 +170,106 @@ async function runWatchdog(req, res) {
 }
 
 // =============================================================================
-// HELPER FUNCTIONS
+// ACTIVE GARBAGE COLLECTION LOGIC
 // =============================================================================
 
-/**
- * Calculates all execution times for a schedule within a start/end window.
- * Returns Array<Date>
- */
-function getOccurrencesInWindow(schedule, start, end) {
-    const times = [];
-    const [h, m] = (schedule.time || '02:00').split(':').map(Number);
-    
-    // Clone start date to iterate
-    let current = new Date(start);
-    current.setUTCHours(h, m, 0, 0);
-
-    // If current is before start (e.g. window starts at 10:00, schedule is 02:00), move to tomorrow
-    if (current < start) {
-        current.setDate(current.getDate() + 1);
-    }
+async function cleanupOrphanedTasks() {
+    const parent = getQueuePath();
+    const validKebabNames = new Set(manifest.map(m => toKebab(m.originalName)));
+    const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
+    let deletedCount = 0;
 
-    while (current <= end) {
-        let match = true;
-        
-        // Weekly Check
-        if (schedule.frequency === 'weekly' && current.getUTCDay() !== (schedule.dayOfWeek ?? 0)) {
-            match = false;
-        }
-        
-        // Monthly Check
-        if (schedule.frequency === 'monthly' && current.getUTCDate() !== (schedule.dayOfMonth ?? 1)) {
-            match = false;
-        }
+    try {
+        // Iterate over ALL tasks in the queue
+        // Note: listTasksAsync handles pagination automatically
+        const tasksToDelete = [];
+
+        for await (const task of tasksClient.listTasksAsync({ parent, responseView: 'BASIC' })) {
+            const taskNameFull = task.name;
+            const taskNameShort = taskNameFull.split('/').pop(); // e.g., root-my-comp-2023-01-01-abcdef
 
-        if (match) {
-            times.push(new Date(current));
+            // 1. Regex Match: Capture the computation name part
+            // Pattern: (root|recovery)-{kebabName}-{date}-{hash}
+            // Date is YYYY-MM-DD (10 chars)
+            // Hash is 8 chars (or more)
+            const match = taskNameShort.match(/^(?:root|recovery)-(.+)-\d{4}-\d{2}-\d{2}-/);
+
+            if (!match) continue; // Skip tasks that don't match our naming convention
+
+            const extractedKebabName = match[1];
+
+            // 2. Check Validity
+            if (!validKebabNames.has(extractedKebabName)) {
+                // ORPHAN DETECTED!
+                tasksToDelete.push(taskNameFull);
+            }
         }
 
-        // Advance 1 day
-        current.setDate(current.getDate() + 1);
+        if (tasksToDelete.length === 0) return 0;
+
+        console.log(`[Planner] 🗑️ Found ${tasksToDelete.length} orphaned tasks. Deleting...`);
+
+        // 3. Delete in parallel
+        await Promise.all(tasksToDelete.map(name => limit(async () => {
+            try {
+                await tasksClient.deleteTask({ name });
+                deletedCount++;
+            } catch (e) {
+                console.warn(`[Planner] Failed to delete orphan ${name}: ${e.message}`);
+            }
+        })));
+
+    } catch (e) {
+        console.error(`[Planner] GC Error: ${e.message}`);
     }
-    
-    return times;
+
+    return deletedCount;
 }
 
-/**
- * Generates a short hash of the Scheduling Config + Pass.
- * If this changes, we want a new Task ID to enforce the new schedule.
- */
-function generateConfigHash(entry) {
-    const input = JSON.stringify(entry.schedule) + `|PASS:${entry.pass}`;
-    return crypto.createHash('md5').update(input).digest('hex').substring(0, 8);
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function shouldRunOnDate(schedule, dateObj) {
+    if (schedule.frequency === 'weekly' && dateObj.getUTCDay() !== (schedule.dayOfWeek ?? 0)) return false;
+    if (schedule.frequency === 'monthly' && dateObj.getUTCDate() !== (schedule.dayOfMonth ?? 1)) return false;
+    return true;
+}
+
+function getRunTimeSeconds(schedule, dateObj) {
+    const [h, m] = (schedule.time || '02:00').split(':').map(Number);
+    const runTime = new Date(dateObj);
+    runTime.setUTCHours(h, m, 0, 0);
+    return runTime.getTime() / 1000;
 }
 
-function getQueuePath(entry) {
+function getQueuePath() {
     const { projectId, location, queueName } = config.cloudTasks;
     return tasksClient.queuePath(projectId, location, queueName);
 }
 
-/**
- * Dispatches Planned Root Tasks
- * Uses deterministic naming for deduplication.
- */
-async function dispatchPlannedTasks(tasks) {
-    const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
-    const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
-
-    return Promise.all(tasks.map(t => limit(async () => {
-        try {
-            // Task Name: root-{name}-{date}-{configHash}
-            // If developer changes schedule -> hash changes -> new task created.
-            // If developer changes code but not schedule -> hash same -> existing task preserved.
-            const taskName = `${t.queuePath}/tasks/root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
-            
-            const payload = {
-                computationName: t.computation,
-                targetDate: t.targetDate,
-                source: 'scheduled',
-                configHash: t.configHash // Sent to dispatcher for potential validation
-            };
-
-            const task = {
-                httpRequest: {
-                    httpMethod: 'POST',
-                    url: dispatcherUrl,
-                    headers: { 'Content-Type': 'application/json' },
-                    body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: { 
-                        serviceAccountEmail,
-                        audience: dispatcherUrl // <--- FIXED: Must exactly match function URL
-                    }
-                },
-                scheduleTime: { seconds: t.runAtSeconds },
-                name: taskName
-            };
-
-            await tasksClient.createTask({ parent: t.queuePath, task });
-            return { computation: t.computation, date: t.targetDate, status: 'scheduled' };
-
-        } catch (e) {
-            if (e.code === 6 || e.code === 409) {
-                return { computation: t.computation, date: t.targetDate, status: 'exists' };
-            }
-            console.error(`[Planner] Failed to schedule ${t.computation}:`, e.message);
-            return { computation: t.computation, status: 'error', error: e.message };
-        }
-    })));
+function toKebab(str) {
+    return str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase().replace(/[^a-z0-9-]/g, '');
 }
 
-/**
- * Dispatches Recovery Tasks (Zombies)
- * Always creates unique task names to ensure retry.
- */
-async function dispatchRecoveryTasks(tasks) {
+async function dispatchTasks(tasks) {
     const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
     const { dispatcherUrl, serviceAccountEmail } = config.cloudTasks;
 
     return Promise.all(tasks.map(t => limit(async () => {
         try {
-            // Unique ID for every recovery attempt
-            const taskName = `${t.queuePath}/tasks/recovery-${toKebab(t.computation)}-${t.recoveryId}-${Date.now()}`;
-            
+            const name = t.isRecovery
+                ? `recovery-${toKebab(t.computation)}-${t.targetDate}-${Date.now()}`
+                : `root-${toKebab(t.computation)}-${t.targetDate}-${t.configHash}`;
+
+            const taskName = `${t.queuePath}/tasks/${name}`;
+
             const payload = {
                 computationName: t.computation,
                 targetDate: t.targetDate,
-                source: 'zombie-recovery'
+                source: t.isRecovery ? 'zombie-recovery' : 'scheduled',
+                configHash: t.configHash,
+                reason: t.reason
             };
 
             const task = {
@@ -304,26 +278,21 @@ async function dispatchRecoveryTasks(tasks) {
                     url: dispatcherUrl,
                     headers: { 'Content-Type': 'application/json' },
                     body: Buffer.from(JSON.stringify(payload)).toString('base64'),
-                    oidcToken: { 
-                        serviceAccountEmail,
-                        audience: dispatcherUrl // <--- FIXED: Must exactly match function URL
-                    }
+                    oidcToken: { serviceAccountEmail, audience: dispatcherUrl }
                 },
-                // Run Immediately (no scheduleTime)
                 name: taskName
             };
 
-            await tasksClient.createTask({ parent: t.queuePath, task });
-            return { computation: t.computation, status: 'recovered' };
+            if (t.runAtSeconds) task.scheduleTime = { seconds: t.runAtSeconds };
 
+            await tasksClient.createTask({ parent: t.queuePath, task });
+            return { status: 'scheduled' };
         } catch (e) {
-            return { computation: t.computation, status: 'error', error: e.message };
+            if (e.code === 6 || e.code === 409) return { status: 'exists' };
+            console.error(`[Planner] Failed task ${t.computation}: ${e.message}`);
+            return { status: 'error' };
         }
     })));
 }
 
-function toKebab(str) {
-    return str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase().replace(/[^a-z0-9-]/g, '');
-}
-
 module.exports = { planComputations, runWatchdog };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.769",
+  "version": "1.0.770",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system-v2/computations/SignedInUserList.js

@@ -1,51 +0,0 @@
-const { Computation } = require('../framework');
-
-class DebugSignedInUsers extends Computation {
-    static getConfig() {
-        return {
-            name: 'DebugSignedInUsers',
-            type: 'per-entity',
-            category: 'debug',
-            
-            // CRITICAL: Tells the Orchestrator which partition to use
-            userType: 'SIGNED_IN_USER', 
-
-            requires: {
-                // DRIVER TABLE: This MUST be first.
-                // The DAG will only create tasks for Entity IDs found in this filtered query.
-                'portfolio_snapshots': {
-                    lookback: 0, // Just need today's existence
-                    mandatory: true,
-                    fields: ['user_id', 'date'],
-                    filter: { user_type: 'SIGNED_IN_USER' }
-                }
-            },
-            
-            storage: {
-                // Just log to console or a debug table for now
-                bigquery: false, 
-                firestore: { enabled: false } 
-            }
-        };
-    }
-
-    async process(context) {
-        const { entityId, data } = context;
-        
-        // If the Orchestrator works correctly, we should ONLY see Signed-In Users here.
-        // We verify by returning their ID.
-        
-        const row = data['portfolio_snapshots'];
-        
-        // Sanity Check: If row exists, we are good.
-        if (row) {
-             this.setResult(entityId, {
-                 status: 'Found',
-                 userType: 'SIGNED_IN_USER',
-                 checkedAt: new Date().toISOString()
-             });
-        }
-    }
-}
-
-module.exports = DebugSignedInUsers;