bulltrackers-module 1.0.772 → 1.0.773

package/functions/computation-system-v2/config/bulltrackers.config.js

@@ -20,7 +20,7 @@ const RESULT_TABLE_NAME = 'computation_results_v3';
 function loadComputations() {
     const computationsDir = path.join(__dirname, '../computations');
     const files = fs.readdirSync(computationsDir);
-    
+
     return files
         .filter(file => file.endsWith('.js') && !file.startsWith('_')) // Skip hidden/test files
         .map(file => {
@@ -38,20 +38,20 @@ module.exports = {
     // =========================================================================
     // PROJECT CONFIGURATION
     // =========================================================================
-    
+
     project: {
         id: 'bulltrackers',
         name: 'BullTrackers Analytics',
         version: '2.0.0'
     },
-    
+
     // System epoch - bump this to force all computations to re-run
-    epoch: 'v2.0.0',
-    
+    epoch: 'v2.0.1',
+
     // =========================================================================
     // BIGQUERY CONFIGURATION
     // =========================================================================
-    
+
     bigquery: {
         projectId: process.env.GCP_PROJECT_ID || 'stocks-12345',
         dataset: process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data',
@@ -62,12 +62,12 @@ module.exports = {
     // =========================================================================
     // GCS CONFIGURATION (NEW: For Batch Loading)
     // =========================================================================
-    
+
     gcs: {
         bucket: process.env.GCS_BUCKET_ID || 'bulltrackers-computation-staging',
         prefix: 'staging'
     },
-    
+
     // =========================================================================
     // TABLE DEFINITIONS
     // =========================================================================
@@ -80,7 +80,7 @@ module.exports = {
     // The framework will automatically discover the full schema from BigQuery.
     // These hints just tell it how to interpret the data.
     // =========================================================================
-    
+
     tables: {
         // User Portfolio Snapshots
         'portfolio_snapshots': {
@@ -89,7 +89,7 @@ module.exports = {
             clusterFields: ['user_type', 'user_id'], // Clustered by user_type, user_id
             description: 'Daily portfolio snapshots for all users'
         },
-        
+
         // User Trade History
         'trade_history_snapshots': {
             dateField: 'date',
@@ -97,7 +97,7 @@ module.exports = {
             clusterFields: ['user_type', 'user_id'], // Clustered by user_type, user_id
             description: 'Daily trade history snapshots'
         },
-        
+
         // Social Posts
         'social_post_snapshots': {
             dateField: 'date',
@@ -105,7 +105,7 @@ module.exports = {
             clusterFields: ['user_type', 'user_id'], // Clustered by user_type, user_id
             description: 'Daily social post snapshots'
         },
-        
+
         // Asset Prices
         'asset_prices': {
             dateField: 'date',
@@ -113,7 +113,7 @@ module.exports = {
             clusterFields: ['ticker', 'instrument_id'], // Clustered by ticker, instrument_id
             description: 'Daily asset prices'
         },
-        
+
         // PI Rankings
         'pi_rankings': {
             dateField: 'date',
@@ -121,7 +121,7 @@ module.exports = {
             clusterFields: ['pi_id', 'category'], // Clustered by pi_id, category
             description: 'Daily PI rankings snapshot'
         },
-        
+
         // PI Master List (not date-partitioned)
         'pi_master_list': {
             dateField: null,  // Not date-partitioned
@@ -129,7 +129,7 @@ module.exports = {
             clusterFields: ['cid'], // Clustered by cid
             description: 'Master list of all Popular Investors'
         },
-        
+
         // PI Ratings
         'pi_ratings': {
             dateField: 'date',
@@ -137,7 +137,7 @@ module.exports = {
             clusterFields: ['pi_id'], // Clustered by pi_id
             description: 'Daily PI ratings'
         },
-        
+
         // PI Page Views
         'pi_page_views': {
             dateField: 'date',
@@ -145,7 +145,7 @@ module.exports = {
             clusterFields: ['pi_id'], // Clustered by pi_id
             description: 'Daily PI page view metrics'
         },
-        
+
         // Watchlist Membership
         'watchlist_membership': {
             dateField: 'date',
@@ -153,7 +153,7 @@ module.exports = {
             clusterFields: ['pi_id'], // Clustered by pi_id
             description: 'Daily watchlist membership counts'
         },
-        
+
         // PI Alert History
         'pi_alert_history': {
             dateField: 'date',
@@ -161,7 +161,7 @@ module.exports = {
             clusterFields: ['pi_id', 'alert_type'], // Clustered by pi_id, alert_type
             description: 'Daily alert trigger history'
         },
-        
+
         // Instrument Insights
         'instrument_insights': {
             dateField: 'date',
@@ -169,7 +169,7 @@ module.exports = {
             // Note: Clustering not specified in metadata provided, assuming similar to others or unclustered
             description: 'Daily instrument insights'
         },
-        
+
         // Ticker Mappings (not date-partitioned)
         'ticker_mappings': {
             dateField: null,
@@ -177,20 +177,20 @@ module.exports = {
             clusterFields: ['instrument_id'], // Clustered by instrument_id
             description: 'Instrument ID to ticker symbol mappings'
         },
-        
+
         // Computation Results
         'computation_results': {
-            tableName: RESULT_TABLE_NAME, 
+            tableName: RESULT_TABLE_NAME,
             dateField: 'date',
             entityField: null,  // Keyed by computation_name
             clusterFields: ['computation_name', 'category'], // Optimization for result lookups
             description: 'Stored computation results'
         },
-        
+
         // Sector Mappings Table
         'sector_mappings': {
             dateField: null,      // Static data
-            entityField: 'symbol', 
+            entityField: 'symbol',
             // Assuming fast lookup on symbol is desired
             description: 'Ticker to Sector mappings migrated from Firestore'
         },
@@ -199,7 +199,7 @@ module.exports = {
         'behavioral_features': {
             tableName: 'daily_behavioral_features',
             dateField: 'date',
-            entityField: 'user_id', 
+            entityField: 'user_id',
             schema: [
                 { name: 'user_id', type: 'STRING' },
                 { name: 'hhi_score', type: 'FLOAT' },
@@ -207,36 +207,36 @@ module.exports = {
             ]
         }
     },
-    
+
     // NEW: Data to load globally for every computation
     // FIX: Define fields to satisfy QueryBuilder safety checks
     referenceData: [
-        { 
-            table: 'sector_mappings', 
+        {
+            table: 'sector_mappings',
             fields: ['symbol', 'sector'] // Adjust these column names if your DB differs
         }
     ],
-    
+
     // =========================================================================
     // RESULT STORAGE CONFIGURATION
     // =========================================================================
-    
+
     resultStore: {
         table: RESULT_TABLE_NAME,
         partitionField: 'date',
         clusterFields: ['computation_name', 'category']
     },
-    
+
     // =========================================================================
     // COMPUTATIONS
     // =========================================================================
-    
+
     computations: loadComputations(),
-    
+
     // =========================================================================
     // PREDEFINED FILTER SETS
     // =========================================================================
-    
+
     filterSets: {
         'popular_investors': {
             user_type: 'POPULAR_INVESTOR'
@@ -248,28 +248,28 @@ module.exports = {
             user_type: ['POPULAR_INVESTOR', 'SIGNED_IN_USER']
         }
     },
-    
+
     // =========================================================================
     // BUSINESS RULES
     // =========================================================================
-    
+
     rules,
-    
+
     // =========================================================================
     // EXECUTION CONFIGURATION
     // =========================================================================
-    
+
     execution: {
         entityConcurrency: 50,
         insertBatchSize: 500,
         fetchBatchSize: 30000,
         maxDependencyEntities: 10000
     },
-    
+
     // =========================================================================
     // SCHEDULING CONFIGURATION
     // =========================================================================
-    
+
     scheduling: {
         default: {
             frequency: 'daily',
@@ -278,39 +278,39 @@ module.exports = {
         },
         dependencyGapMinutes: 5
     },
-    
+
     // =========================================================================
     // CLOUD TASKS CONFIGURATION
     // =========================================================================
-    
+
     cloudTasks: {
         projectId: process.env.GCP_PROJECT_ID || 'stocks-12345',
         location: 'europe-west1',
         queueName: 'computation-triggers',
-        dispatcherUrl: process.env.DISPATCHER_URL || 
+        dispatcherUrl: process.env.DISPATCHER_URL ||
             'https://europe-west1-stocks-12345.cloudfunctions.net/compute-dispatcher',
-        serviceAccountEmail: process.env.CLOUD_TASKS_SA_EMAIL || 
+        serviceAccountEmail: process.env.CLOUD_TASKS_SA_EMAIL ||
             '879684846540-compute@developer.gserviceaccount.com'
     },
-    
+
     // =========================================================================
     // ON-DEMAND API CONFIGURATION
     // =========================================================================
-    
+
     onDemand: {
         maxRequestsPerMinute: 5,
         timeout: 60000,
         allowedComputations: null
     },
-    
+
     // =========================================================================
     // WORKER POOL CONFIGURATION (SERVERLESS WORKERS)
     // =========================================================================
-    
+
     workerPool: {
         enabled: process.env.WORKER_POOL_ENABLED === 'true',
         localMode: process.env.WORKER_LOCAL_MODE === 'true',
-        workerUrl: process.env.WORKER_URL || 
+        workerUrl: process.env.WORKER_URL ||
             'https://europe-west1-stocks-12345.cloudfunctions.net/computation-worker',
         tempBucket: process.env.WORKER_TEMP_BUCKET || 'bulltrackers-worker-staging',
         concurrency: 100,
@@ -318,8 +318,8 @@ module.exports = {
         retries: 2,
         minEntitiesForOffload: 100, // Fixed duplicate key issue (removed the lower value)
         excludeComputations: [],
-        forceOffloadComputations: process.env.WORKER_FORCE_COMPUTATIONS 
-            ? process.env.WORKER_FORCE_COMPUTATIONS.split(',') 
+        forceOffloadComputations: process.env.WORKER_FORCE_COMPUTATIONS
+            ? process.env.WORKER_FORCE_COMPUTATIONS.split(',')
             : [],
     }
 };

package/functions/computation-system-v2/handlers/scheduler.js

@@ -41,16 +41,14 @@ async function initialize() {
 
     console.log(`[Scheduler] Loaded ${manifest.length} computations.`);
 }
-
 /**
- * ENTRY POINT 1: The Reconciler & Garbage Collector
- * Trigger: Cloud Scheduler -> "0 * * * *" (Every Hour)
+ * REPLACEMENT for planComputations
  */
 async function planComputations(req, res) {
     try {
         await initialize();
 
-        // --- PHASE 1: RECONCILIATION (Ensure valid tasks exist) ---
+        // --- PHASE 1: RECONCILIATION ---
         const now = new Date();
         const windowStart = new Date(now);
         windowStart.setDate(now.getDate() - PLANNING_LOOKBACK_DAYS);
@@ -61,10 +59,21 @@ async function planComputations(req, res) {
 
         console.log(`[Planner] Reconciling window: ${windowStart.toISOString()} to ${windowEnd.toISOString()}`);
 
-        const tasksToSchedule = [];
+        // Helper to find Roots for any given computation (Pass 1..N)
+        const manifestMap = new Map(manifest.map(m => [m.name, m]));
+        const getRoots = (entry, visited = new Set()) => {
+            if (visited.has(entry.name)) return [];
+            visited.add(entry.name);
+            if (entry.pass === 1) return [entry];
+            return (entry.dependencies || [])
+                .map(d => manifestMap.get(d))
+                .filter(Boolean)
+                .flatMap(p => getRoots(p, visited));
+        };
+
+        const tasksToSchedule = new Map(); // Use Map to deduplicate by Task Name
         const stats = { checked: 0, scheduled: 0, mismatched: 0, missing: 0 };
 
-        // Iterate dates in window
         const targetDates = [];
         let cursor = new Date(windowStart);
         while (cursor <= windowEnd) {
@@ -77,8 +86,9 @@ async function planComputations(req, res) {
             const dateStr = dateObj.toISOString().split('T')[0];
             const dailyStatus = await stateRepository.getDailyStatus(dateStr);
 
+            // Iterate ALL computations (not just Pass 1) to find stale nodes
             for (const entry of manifest) {
-                if (entry.pass !== 1) continue; // Only schedule Roots
+                // If this specific entry is not scheduled for today, skip it
                 if (!shouldRunOnDate(entry.schedule, dateObj)) continue;
 
                 stats.checked++;
@@ -94,26 +104,37 @@ async function planComputations(req, res) {
                 }
 
                 if (reason) {
-                    tasksToSchedule.push({
-                        computation: entry.originalName,
-                        targetDate: dateStr,
-                        runAtSeconds: getRunTimeSeconds(entry.schedule, dateObj),
-                        configHash: entry.hash,
-                        queuePath: getQueuePath(),
-                        reason
+                    // If entry is stale, we must schedule its ROOT(s) to trigger the chain
+                    const roots = getRoots(entry);
+
+                    roots.forEach(root => {
+                        const taskKey = `root-${toKebab(root.originalName)}-${dateStr}-${root.hash}`;
+
+                        if (!tasksToSchedule.has(taskKey)) {
+                            tasksToSchedule.set(taskKey, {
+                                computation: root.originalName,
+                                targetDate: dateStr,
+                                runAtSeconds: getRunTimeSeconds(root.schedule, dateObj),
+                                configHash: root.hash,
+                                queuePath: getQueuePath(),
+                                reason: `TRIGGERED_BY_${entry.name}_${reason}` // Track what triggered this root
+                            });
+                        }
                     });
                 }
             }
         })));
 
-        // --- PHASE 2: GARBAGE COLLECTION (Remove invalid tasks) ---
+        // --- PHASE 2: GARBAGE COLLECTION ---
+        // (Keep your existing GC logic here)
         console.log('[Planner] Starting Garbage Collection...');
         const deletedCount = await cleanupOrphanedTasks();
 
         // --- PHASE 3: DISPATCH ---
+        const taskList = Array.from(tasksToSchedule.values());
         let scheduledCount = 0;
-        if (tasksToSchedule.length > 0) {
-            const results = await dispatchTasks(tasksToSchedule);
+        if (taskList.length > 0) {
+            const results = await dispatchTasks(taskList);
             scheduledCount = results.filter(r => r.status === 'scheduled').length;
         }
 
@@ -121,9 +142,8 @@ async function planComputations(req, res) {
 
         return res.status(200).json({
             status: 'success',
-            window: `${PLANNING_LOOKBACK_DAYS}d back, ${PLANNING_LOOKAHEAD_HOURS}h fwd`,
+            window: `${PLANNING_LOOKBACK_DAYS}d back`,
             scheduled: scheduledCount,
-            deletedOrphans: deletedCount,
             stats
         });
 
@@ -174,7 +194,7 @@ async function runWatchdog(req, res) {
 
 async function cleanupOrphanedTasks() {
     const parent = getQueuePath();
-    
+
     // Create a map of { kebabName: activeHash } for O(1) lookups
     const activeComputations = new Map(
         manifest.map(m => [toKebab(m.originalName), m.hash])
@@ -188,13 +208,13 @@ async function cleanupOrphanedTasks() {
 
         // Note: listTasksAsync handles pagination, but if you have thousands of tasks,
         // you might eventually need to handle page tokens explicitly if the library version is old.
-        for await (const task of tasksClient.listTasksAsync({ 
-            parent, 
+        for await (const task of tasksClient.listTasksAsync({
+            parent,
             responseView: 'BASIC',
             pageSize: 1000 // Increase page size to capture more per request
-        })){
+        })) {
             const taskNameFull = task.name;
-            const taskNameShort = taskNameFull.split('/').pop(); 
+            const taskNameShort = taskNameFull.split('/').pop();
 
             // 1. Handle ROOT Tasks: root-{kebabName}-{date}-{hash}
             // We capture the name AND the hash at the end
@@ -217,7 +237,7 @@ async function cleanupOrphanedTasks() {
             // We only delete these if the computation is completely gone.
             // (Timestamps won't match a config hash, so we just check existence)
             const recoveryMatch = taskNameShort.match(/^recovery-(.+)-\d{4}-\d{2}-\d{2}-/);
-            
+
             if (recoveryMatch) {
                 const [_, kebabName] = recoveryMatch;
                 if (!activeComputations.has(kebabName)) {
@@ -237,7 +257,7 @@ async function cleanupOrphanedTasks() {
                 deletedCount++;
             } catch (e) {
                 // Ignore "NOT_FOUND" errors in case of race conditions
-                if (e.code !== 5) { 
+                if (e.code !== 5) {
                     console.warn(`[Planner] Failed to delete ${name}: ${e.message}`);
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.772",
+  "version": "1.0.773",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [