bulltrackers-module 1.0.771 → 1.0.773

package/functions/computation-system-v2/config/bulltrackers.config.js

@@ -20,7 +20,7 @@ const RESULT_TABLE_NAME = 'computation_results_v3';
 function loadComputations() {
     const computationsDir = path.join(__dirname, '../computations');
     const files = fs.readdirSync(computationsDir);
-    
+
     return files
         .filter(file => file.endsWith('.js') && !file.startsWith('_')) // Skip hidden/test files
         .map(file => {
@@ -38,20 +38,20 @@ module.exports = {
     // =========================================================================
     // PROJECT CONFIGURATION
     // =========================================================================
-    
+
     project: {
         id: 'bulltrackers',
         name: 'BullTrackers Analytics',
         version: '2.0.0'
     },
-    
+
     // System epoch - bump this to force all computations to re-run
-    epoch: 'v2.0.0',
-    
+    epoch: 'v2.0.1',
+
     // =========================================================================
     // BIGQUERY CONFIGURATION
     // =========================================================================
-    
+
     bigquery: {
         projectId: process.env.GCP_PROJECT_ID || 'stocks-12345',
         dataset: process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data',
@@ -62,12 +62,12 @@ module.exports = {
     // =========================================================================
     // GCS CONFIGURATION (NEW: For Batch Loading)
     // =========================================================================
-    
+
     gcs: {
         bucket: process.env.GCS_BUCKET_ID || 'bulltrackers-computation-staging',
         prefix: 'staging'
     },
-    
+
     // =========================================================================
     // TABLE DEFINITIONS
     // =========================================================================
@@ -80,7 +80,7 @@ module.exports = {
     // The framework will automatically discover the full schema from BigQuery.
     // These hints just tell it how to interpret the data.
     // =========================================================================
-    
+
     tables: {
         // User Portfolio Snapshots
         'portfolio_snapshots': {
@@ -89,7 +89,7 @@ module.exports = {
             clusterFields: ['user_type', 'user_id'], // Clustered by user_type, user_id
             description: 'Daily portfolio snapshots for all users'
         },
-        
+
         // User Trade History
         'trade_history_snapshots': {
             dateField: 'date',
@@ -97,7 +97,7 @@ module.exports = {
             clusterFields: ['user_type', 'user_id'], // Clustered by user_type, user_id
             description: 'Daily trade history snapshots'
         },
-        
+
         // Social Posts
         'social_post_snapshots': {
             dateField: 'date',
@@ -105,7 +105,7 @@ module.exports = {
             clusterFields: ['user_type', 'user_id'], // Clustered by user_type, user_id
             description: 'Daily social post snapshots'
         },
-        
+
         // Asset Prices
         'asset_prices': {
             dateField: 'date',
@@ -113,7 +113,7 @@ module.exports = {
             clusterFields: ['ticker', 'instrument_id'], // Clustered by ticker, instrument_id
             description: 'Daily asset prices'
         },
-        
+
         // PI Rankings
         'pi_rankings': {
             dateField: 'date',
@@ -121,7 +121,7 @@ module.exports = {
             clusterFields: ['pi_id', 'category'], // Clustered by pi_id, category
             description: 'Daily PI rankings snapshot'
         },
-        
+
         // PI Master List (not date-partitioned)
         'pi_master_list': {
             dateField: null,  // Not date-partitioned
@@ -129,7 +129,7 @@ module.exports = {
             clusterFields: ['cid'], // Clustered by cid
             description: 'Master list of all Popular Investors'
         },
-        
+
         // PI Ratings
         'pi_ratings': {
             dateField: 'date',
@@ -137,7 +137,7 @@ module.exports = {
             clusterFields: ['pi_id'], // Clustered by pi_id
             description: 'Daily PI ratings'
         },
-        
+
         // PI Page Views
         'pi_page_views': {
             dateField: 'date',
@@ -145,7 +145,7 @@ module.exports = {
             clusterFields: ['pi_id'], // Clustered by pi_id
             description: 'Daily PI page view metrics'
         },
-        
+
         // Watchlist Membership
         'watchlist_membership': {
             dateField: 'date',
@@ -153,7 +153,7 @@ module.exports = {
             clusterFields: ['pi_id'], // Clustered by pi_id
             description: 'Daily watchlist membership counts'
         },
-        
+
         // PI Alert History
         'pi_alert_history': {
             dateField: 'date',
@@ -161,7 +161,7 @@ module.exports = {
             clusterFields: ['pi_id', 'alert_type'], // Clustered by pi_id, alert_type
             description: 'Daily alert trigger history'
         },
-        
+
         // Instrument Insights
         'instrument_insights': {
             dateField: 'date',
@@ -169,7 +169,7 @@ module.exports = {
             // Note: Clustering not specified in metadata provided, assuming similar to others or unclustered
             description: 'Daily instrument insights'
         },
-        
+
         // Ticker Mappings (not date-partitioned)
         'ticker_mappings': {
             dateField: null,
@@ -177,20 +177,20 @@ module.exports = {
             clusterFields: ['instrument_id'], // Clustered by instrument_id
             description: 'Instrument ID to ticker symbol mappings'
         },
-        
+
         // Computation Results
         'computation_results': {
-            tableName: RESULT_TABLE_NAME, 
+            tableName: RESULT_TABLE_NAME,
             dateField: 'date',
             entityField: null,  // Keyed by computation_name
             clusterFields: ['computation_name', 'category'], // Optimization for result lookups
             description: 'Stored computation results'
         },
-        
+
         // Sector Mappings Table
         'sector_mappings': {
             dateField: null,      // Static data
-            entityField: 'symbol', 
+            entityField: 'symbol',
             // Assuming fast lookup on symbol is desired
             description: 'Ticker to Sector mappings migrated from Firestore'
         },
@@ -199,7 +199,7 @@ module.exports = {
         'behavioral_features': {
             tableName: 'daily_behavioral_features',
             dateField: 'date',
-            entityField: 'user_id', 
+            entityField: 'user_id',
             schema: [
                 { name: 'user_id', type: 'STRING' },
                 { name: 'hhi_score', type: 'FLOAT' },
@@ -207,36 +207,36 @@ module.exports = {
             ]
         }
     },
-    
+
     // NEW: Data to load globally for every computation
     // FIX: Define fields to satisfy QueryBuilder safety checks
     referenceData: [
-        { 
-            table: 'sector_mappings', 
+        {
+            table: 'sector_mappings',
             fields: ['symbol', 'sector'] // Adjust these column names if your DB differs
         }
     ],
-    
+
     // =========================================================================
     // RESULT STORAGE CONFIGURATION
     // =========================================================================
-    
+
     resultStore: {
         table: RESULT_TABLE_NAME,
         partitionField: 'date',
         clusterFields: ['computation_name', 'category']
     },
-    
+
     // =========================================================================
     // COMPUTATIONS
     // =========================================================================
-    
+
     computations: loadComputations(),
-    
+
     // =========================================================================
     // PREDEFINED FILTER SETS
     // =========================================================================
-    
+
     filterSets: {
         'popular_investors': {
             user_type: 'POPULAR_INVESTOR'
@@ -248,28 +248,28 @@ module.exports = {
             user_type: ['POPULAR_INVESTOR', 'SIGNED_IN_USER']
         }
     },
-    
+
     // =========================================================================
     // BUSINESS RULES
     // =========================================================================
-    
+
     rules,
-    
+
     // =========================================================================
     // EXECUTION CONFIGURATION
     // =========================================================================
-    
+
     execution: {
         entityConcurrency: 50,
         insertBatchSize: 500,
         fetchBatchSize: 30000,
         maxDependencyEntities: 10000
     },
-    
+
     // =========================================================================
     // SCHEDULING CONFIGURATION
     // =========================================================================
-    
+
     scheduling: {
         default: {
             frequency: 'daily',
@@ -278,39 +278,39 @@ module.exports = {
         },
         dependencyGapMinutes: 5
     },
-    
+
     // =========================================================================
     // CLOUD TASKS CONFIGURATION
     // =========================================================================
-    
+
     cloudTasks: {
         projectId: process.env.GCP_PROJECT_ID || 'stocks-12345',
         location: 'europe-west1',
         queueName: 'computation-triggers',
-        dispatcherUrl: process.env.DISPATCHER_URL || 
+        dispatcherUrl: process.env.DISPATCHER_URL ||
             'https://europe-west1-stocks-12345.cloudfunctions.net/compute-dispatcher',
-        serviceAccountEmail: process.env.CLOUD_TASKS_SA_EMAIL || 
+        serviceAccountEmail: process.env.CLOUD_TASKS_SA_EMAIL ||
             '879684846540-compute@developer.gserviceaccount.com'
     },
-    
+
     // =========================================================================
     // ON-DEMAND API CONFIGURATION
     // =========================================================================
-    
+
     onDemand: {
         maxRequestsPerMinute: 5,
         timeout: 60000,
         allowedComputations: null
     },
-    
+
     // =========================================================================
     // WORKER POOL CONFIGURATION (SERVERLESS WORKERS)
     // =========================================================================
-    
+
     workerPool: {
         enabled: process.env.WORKER_POOL_ENABLED === 'true',
         localMode: process.env.WORKER_LOCAL_MODE === 'true',
-        workerUrl: process.env.WORKER_URL || 
+        workerUrl: process.env.WORKER_URL ||
             'https://europe-west1-stocks-12345.cloudfunctions.net/computation-worker',
         tempBucket: process.env.WORKER_TEMP_BUCKET || 'bulltrackers-worker-staging',
         concurrency: 100,
@@ -318,8 +318,8 @@ module.exports = {
         retries: 2,
         minEntitiesForOffload: 100, // Fixed duplicate key issue (removed the lower value)
         excludeComputations: [],
-        forceOffloadComputations: process.env.WORKER_FORCE_COMPUTATIONS 
-            ? process.env.WORKER_FORCE_COMPUTATIONS.split(',') 
+        forceOffloadComputations: process.env.WORKER_FORCE_COMPUTATIONS
+            ? process.env.WORKER_FORCE_COMPUTATIONS.split(',')
             : [],
     }
 };

package/functions/computation-system-v2/handlers/scheduler.js

@@ -41,16 +41,14 @@ async function initialize() {
 
     console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
 }
-
 /**
- * ENTRY POINT 1: The Reconciler & Garbage Collector
- * Trigger: Cloud Scheduler -> "0 * * * *" (Every Hour)
+ * REPLACEMENT for planComputations
  */
 async function planComputations(req, res) {
     try {
         await initialize();
 
-        // --- PHASE 1: RECONCILIATION (Ensure valid tasks exist) ---
+        // --- PHASE 1: RECONCILIATION ---
         const now = new Date();
         const windowStart = new Date(now);
         windowStart.setDate(now.getDate() - PLANNING_LOOKBACK_DAYS);
@@ -61,10 +59,21 @@ async function planComputations(req, res) {
 
         console.log(`[Planner] Reconciling window: ${windowStart.toISOString()} to ${windowEnd.toISOString()}`);
 
-        const tasksToSchedule = [];
+        // Helper to find Roots for any given computation (Pass 1..N)
+        const manifestMap = new Map(manifest.map(m => [m.name, m]));
+        const getRoots = (entry, visited = new Set()) => {
+            if (visited.has(entry.name)) return [];
+            visited.add(entry.name);
+            if (entry.pass === 1) return [entry];
+            return (entry.dependencies || [])
+                .map(d => manifestMap.get(d))
+                .filter(Boolean)
+                .flatMap(p => getRoots(p, visited));
+        };
+
+        const tasksToSchedule = new Map(); // Use Map to deduplicate by Task Name
         const stats = { checked: 0, scheduled: 0, mismatched: 0, missing: 0 };
 
-        // Iterate dates in window
         const targetDates = [];
         let cursor = new Date(windowStart);
         while (cursor <= windowEnd) {
@@ -77,8 +86,9 @@ async function planComputations(req, res) {
             const dateStr = dateObj.toISOString().split('T')[0];
             const dailyStatus = await stateRepository.getDailyStatus(dateStr);
 
+            // Iterate ALL computations (not just Pass 1) to find stale nodes
             for (const entry of manifest) {
-                if (entry.pass !== 1) continue; // Only schedule Roots
+                // If this specific entry is not scheduled for today, skip it
                 if (!shouldRunOnDate(entry.schedule, dateObj)) continue;
 
                 stats.checked++;
@@ -94,26 +104,37 @@ async function planComputations(req, res) {
                 }
 
                 if (reason) {
-                    tasksToSchedule.push({
-                        computation: entry.originalName,
-                        targetDate: dateStr,
-                        runAtSeconds: getRunTimeSeconds(entry.schedule, dateObj),
-                        configHash: entry.hash,
-                        queuePath: getQueuePath(),
-                        reason
+                    // If entry is stale, we must schedule its ROOT(s) to trigger the chain
+                    const roots = getRoots(entry);
+
+                    roots.forEach(root => {
+                        const taskKey = `root-${toKebab(root.originalName)}-${dateStr}-${root.hash}`;
+
+                        if (!tasksToSchedule.has(taskKey)) {
+                            tasksToSchedule.set(taskKey, {
+                                computation: root.originalName,
+                                targetDate: dateStr,
+                                runAtSeconds: getRunTimeSeconds(root.schedule, dateObj),
+                                configHash: root.hash,
+                                queuePath: getQueuePath(),
+                                reason: `TRIGGERED_BY_${entry.name}_${reason}` // Track what triggered this root
+                            });
+                        }
                     });
                 }
             }
         })));
 
-        // --- PHASE 2: GARBAGE COLLECTION (Remove invalid tasks) ---
+        // --- PHASE 2: GARBAGE COLLECTION ---
+        // (Keep your existing GC logic here)
         console.log('[Planner] Starting Garbage Collection...');
         const deletedCount = await cleanupOrphanedTasks();
 
         // --- PHASE 3: DISPATCH ---
+        const taskList = Array.from(tasksToSchedule.values());
         let scheduledCount = 0;
-        if (tasksToSchedule.length > 0) {
-            const results = await dispatchTasks(tasksToSchedule);
+        if (taskList.length > 0) {
+            const results = await dispatchTasks(taskList);
             scheduledCount = results.filter(r => r.status === 'scheduled').length;
         }
 
@@ -121,9 +142,8 @@ async function planComputations(req, res) {
 
         return res.status(200).json({
             status: 'success',
-            window: `${PLANNING_LOOKBACK_DAYS}d back, ${PLANNING_LOOKAHEAD_HOURS}h fwd`,
+            window: `${PLANNING_LOOKBACK_DAYS}d back`,
             scheduled: scheduledCount,
-            deletedOrphans: deletedCount,
             stats
         });
 
@@ -168,46 +188,67 @@ async function runWatchdog(req, res) {
         return res.status(500).json({ error: error.message });
     }
 }
-
 // =============================================================================
 // ACTIVE GARBAGE COLLECTION LOGIC
 // =============================================================================
 
 async function cleanupOrphanedTasks() {
     const parent = getQueuePath();
-    const validKebabNames = new Set(manifest.map(m => toKebab(m.originalName)));
+
+    // Create a map of { kebabName: activeHash } for O(1) lookups
+    const activeComputations = new Map(
+        manifest.map(m => [toKebab(m.originalName), m.hash])
+    );
+
     const limit = pLimit(CLOUD_TASKS_CONCURRENCY);
     let deletedCount = 0;
 
     try {
-        // Iterate over ALL tasks in the queue
-        // Note: listTasksAsync handles pagination automatically
         const tasksToDelete = [];
 
-        for await (const task of tasksClient.listTasksAsync({ parent, responseView: 'BASIC' })) {
+        // Note: listTasksAsync handles pagination, but if you have thousands of tasks,
+        // you might eventually need to handle page tokens explicitly if the library version is old.
+        for await (const task of tasksClient.listTasksAsync({
+            parent,
+            responseView: 'BASIC',
+            pageSize: 1000 // Increase page size to capture more per request
+        })) {
             const taskNameFull = task.name;
-            const taskNameShort = taskNameFull.split('/').pop(); // e.g., root-my-comp-2023-01-01-abcdef
+            const taskNameShort = taskNameFull.split('/').pop();
 
-            // 1. Regex Match: Capture the computation name part
-            // Pattern: (root|recovery)-{kebabName}-{date}-{hash}
-            // Date is YYYY-MM-DD (10 chars)
-            // Hash is 8 chars (or more)
-            const match = taskNameShort.match(/^(?:root|recovery)-(.+)-\d{4}-\d{2}-\d{2}-/);
+            // 1. Handle ROOT Tasks: root-{kebabName}-{date}-{hash}
+            // We capture the name AND the hash at the end
+            const rootMatch = taskNameShort.match(/^root-(.+)-\d{4}-\d{2}-\d{2}-(.+)$/);
 
-            if (!match) continue; // Skip tasks that don't match our naming convention
+            if (rootMatch) {
+                const [_, kebabName, taskHash] = rootMatch;
+                const activeHash = activeComputations.get(kebabName);
 
-            const extractedKebabName = match[1];
+                // DELETE IF:
+                // A) Computation removed from manifest (!activeHash)
+                // B) Hash mismatch (Old deployment/Stale) (activeHash !== taskHash)
+                if (!activeHash || activeHash !== taskHash) {
+                    tasksToDelete.push(taskNameFull);
+                }
+                continue;
+            }
+
+            // 2. Handle RECOVERY Tasks: recovery-{kebabName}-{date}-{timestamp}
+            // We only delete these if the computation is completely gone.
+            // (Timestamps won't match a config hash, so we just check existence)
+            const recoveryMatch = taskNameShort.match(/^recovery-(.+)-\d{4}-\d{2}-\d{2}-/);
 
-            // 2. Check Validity
-            if (!validKebabNames.has(extractedKebabName)) {
-                // ORPHAN DETECTED!
-                tasksToDelete.push(taskNameFull);
+            if (recoveryMatch) {
+                const [_, kebabName] = recoveryMatch;
+                if (!activeComputations.has(kebabName)) {
+                    tasksToDelete.push(taskNameFull);
+                }
             }
         }
 
         if (tasksToDelete.length === 0) return 0;
 
-        console.log(`[Planner] 🗑️ Found ${tasksToDelete.length} orphaned tasks. Deleting...`);
+        console.log(`[Planner] 🗑️ Found ${tasksToDelete.length} stale/orphaned tasks. Deleting...`);
 
         // 3. Delete in parallel
         await Promise.all(tasksToDelete.map(name => limit(async () => {
@@ -215,7 +256,10 @@ async function cleanupOrphanedTasks() {
                 await tasksClient.deleteTask({ name });
                 deletedCount++;
             } catch (e) {
-                console.warn(`[Planner] Failed to delete orphan ${name}: ${e.message}`);
+                // Ignore "NOT_FOUND" errors in case of race conditions
+                if (e.code !== 5) {
+                    console.warn(`[Planner] Failed to delete ${name}: ${e.message}`);
+                }
             }
         })));
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.771",
+  "version": "1.0.773",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [