bulltrackers-module 1.0.431 → 1.0.433

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,8 +1,6 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
- * UPDATED: Tracks specific Firestore Ops (Writes/Deletes) for cost analysis.
- * UPDATED: Added Dynamic Circuit Breaker Bypass for Price-Only calculations.
- * UPDATED: Integrated Alert System Pub/Sub triggers for marked computations.
+ * UPDATED: Fixed bug where Alert Computations failed to trigger Pub/Sub on empty FINAL flush.
  */
 const { commitBatchInChunks, generateDataHash } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -24,7 +22,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     const failureReport  = [];
     const schemas        = [];
     const cleanupTasks   = []; 
-    const alertTriggers  = []; // [NEW] Collect alert triggers
+    const alertTriggers  = []; 
     const { logger, db } = deps;
     const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
     
@@ -35,7 +33,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     const fanOutLimit      = pLimit(10); 
     const pubSubUtils      = new PubSubUtils(deps);
 
-    // 1. [BATCH OPTIMIZATION] Fetch all SimHashes and Contracts upfront
     const calcNames = Object.keys(stateObj);
     const hashKeys  = calcNames.map(n => stateObj[n].manifest?.hash).filter(Boolean);
     
@@ -56,11 +53,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             io: { writes: 0, deletes: 0 }
         };
 
+        // [NEW] Check metadata for alert flag (defaults to false)
+        const isAlertComputation = calc.manifest.isAlertComputation === true;
+
         try {
             const result = await calc.getResult();
             const configOverrides = validationOverrides[calc.manifest.name] || {};
             
-            // --- [DYNAMIC OVERRIDE LOGIC START] ---
             const dataDeps = calc.manifest.rootDataDependencies || [];
             const isPriceOnly = (dataDeps.length === 1 && dataDeps[0] === 'price');
             let effectiveOverrides = { ...configOverrides };
@@ -72,7 +71,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 effectiveOverrides.maxNanPct = 100;
                 delete effectiveOverrides.weekend; 
             }
-            // --- [DYNAMIC OVERRIDE LOGIC END] ---
 
             const contract = contractMap[name];
             if (contract) {
@@ -99,11 +97,27 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0);
             const resultHash = isEmpty ? 'empty' : generateDataHash(result);
-            
             const simHash = (flushMode !== 'INTERMEDIATE') ? (simHashMap[calc.manifest.hash] || null) : null;
 
             if (isEmpty) { 
-                if (flushMode === 'INTERMEDIATE') { nextShardIndexes[name] = currentShardIndex; continue; }
+                if (flushMode === 'INTERMEDIATE') { 
+                    nextShardIndexes[name] = currentShardIndex; 
+                    continue; 
+                }
+                
+                // [FIX] Force alert trigger on FINAL flush even if result is empty
+                // This handles cases where all data was written in previous INTERMEDIATE flushes
+                if (isAlertComputation && flushMode === 'FINAL') {
+                    // Reconstruct path to ensure downstream system checks for data
+                    const docPath = `${config.resultsCollection}/${dStr}/${config.resultsSubcollection}/${calc.manifest.category}/${config.computationsSubcollection}/${name}`;
+                    
+                    alertTriggers.push({
+                        date: dStr,
+                        computationName: name,
+                        documentPath: docPath
+                    });
+                }
+
                 if (calc.manifest.hash) { 
                     successUpdates[name] = { 
                         hash: calc.manifest.hash, simHash: simHash, resultHash: resultHash, 
@@ -119,9 +133,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const resultKeys = Object.keys(result || {});
             const isMultiDate = resultKeys.length > 0 && resultKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k));
 
-            // [NEW] Check metadata for alert flag (defaults to false)
-            const isAlertComputation = calc.manifest.isAlertComputation === true;
-
             if (isMultiDate) {
                 const datePromises  = resultKeys.map((historicalDate) => fanOutLimit(async () => {
                     const dailyData = result[historicalDate];
@@ -131,7 +142,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     runMetrics.io.writes += stats.opCounts.writes;
                     runMetrics.io.deletes += stats.opCounts.deletes;
 
-                    // [NEW] Queue alert for historical date if applicable
                     if (isAlertComputation && flushMode !== 'INTERMEDIATE') {
                         alertTriggers.push({
                             date: historicalDate,
@@ -156,7 +166,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 nextShardIndexes[name] = writeStats.nextShardIndex;
                 if (calc.manifest.hash) { successUpdates[name] = { hash: calc.manifest.hash, simHash, resultHash, dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, category: calc.manifest.category, composition: calc.manifest.composition, metrics: runMetrics }; }
 
-                // [NEW] Queue alert for standard write
                 if (isAlertComputation && flushMode !== 'INTERMEDIATE') {
                     alertTriggers.push({
                         date: dStr,
@@ -193,7 +202,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         await updateComputationStatus(dStr, successUpdates, config, deps); 
     }
 
-    // [NEW] Flush Alert Triggers to Pub/Sub
+    // Flush Alert Triggers to Pub/Sub
     if (alertTriggers.length > 0) {
         const topicName = config.alertTopicName || 'alert-trigger';
         try {
@@ -201,7 +210,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             logger.log('INFO', `[Alert System] Triggered ${alertTriggers.length} alerts to ${topicName}`);
         } catch (alertErr) {
             logger.log('ERROR', `[Alert System] Failed to publish alerts: ${alertErr.message}`);
-            // Non-blocking: We don't fail the computation if alerts fail to publish, but we log strictly.
         }
     }
     

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -0,0 +1,140 @@
+# Data Feeder Pipeline
+# Orchestrates data fetching relative to Market Close (22:00 UTC) and Midnight (00:00 UTC).
+# Triggers at 22:00 UTC.
+
+main:
+  params: [input]
+  steps:
+    - init:
+        assign:
+          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
+          - location: "europe-west1"
+          - market_date: '${text.split(time.format(sys.now()), "T")[0]}'
+
+    # --- PHASE 1: MARKET CLOSE (22:00 UTC) ---
+    - run_market_close_tasks:
+        parallel:
+          branches:
+            - price_fetch:
+                steps:
+                  - call_price_fetcher:
+                      call: http.post
+                      args:
+                        url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
+                        auth: { type: OIDC }
+            - insights_fetch:
+                steps:
+                  - call_insights_fetcher:
+                      call: http.post
+                      args:
+                        url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
+                        auth: { type: OIDC }
+
+    # [IMMEDIATE INDEX: MARKET DATA]
+    # Update index for the market date we just fetched
+    - index_market_data:
+        call: http.post
+        args:
+          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+          body:
+            targetDate: '${market_date}'
+          auth: { type: OIDC }
+
+    # --- PHASE 2: WAIT FOR MIDNIGHT ---
+    # Trigger is at 22:00. We wait ~2 hours to align with 00:00 UTC.
+    - wait_for_midnight:
+        call: sys.sleep
+        args:
+          seconds: 7200 # 2 Hours
+
+    # --- PHASE 3: RANKINGS & INITIAL SOCIAL (00:00 UTC) ---
+    # Rankings must run first.
+    - run_rankings_fetch:
+        try:
+          call: http.post
+          args:
+            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
+            auth: { type: OIDC }
+        except:
+          as: e
+          steps:
+            - log_rankings_error:
+                call: sys.log
+                args:
+                  severity: "ERROR"
+                  text: '${"Rankings Fetch Failed. Proceeding to Social Fetch. Error: " + json.encode(e)}'
+
+    - run_social_midnight:
+        call: http.post
+        args:
+          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
+          auth: { type: OIDC }
+
+    # [IMMEDIATE INDEX: MIDNIGHT DATA]
+    # We recalculate the date because we are now past midnight
+    - index_midnight_data:
+        assign:
+            - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
+        next: call_index_midnight
+
+    - call_index_midnight:
+        call: http.post
+        args:
+          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+          body:
+             targetDate: '${current_date}'
+          auth: { type: OIDC }
+
+    # --- PHASE 4: GLOBAL SYNC (Before 01:00 UTC) ---
+    # Critical: Indexes EVERYTHING (no targetDate) to ensure the Computation System 
+    # (which starts at 01:00 UTC) has a fully consistent view of history.
+    - run_global_indexer:
+        call: http.post
+        args:
+          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+          # No body = Full Scan
+          auth: { type: OIDC }
+        result: global_index_res
+
+    # --- PHASE 5: RECURRING SOCIAL FETCH (Every 3 Hours) ---
+    # We loop to cover the rest of the 24h cycle.
+    - init_social_loop:
+        assign:
+          - i: 0
+    
+    - social_loop:
+        switch:
+          - condition: ${i < 7}
+            steps:
+              - wait_3_hours:
+                  call: sys.sleep
+                  args:
+                    seconds: 10800 # 3 hours
+              
+              - run_social_recurring:
+                  call: http.post
+                  args:
+                    url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
+                    auth: { type: OIDC }
+
+              # [IMMEDIATE INDEX: RECURRING]
+              - calculate_recurring_date:
+                  assign:
+                    - current_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
+              
+              - index_recurring:
+                   call: http.post
+                   args:
+                     url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+                     body:
+                       targetDate: '${current_date_rec}'
+                     auth: { type: OIDC }
+              
+              - increment:
+                  assign:
+                    - i: ${i + 1}
+              - next_iteration:
+                  next: social_loop
+
+    - finish:
+        return: "Daily Data Pipeline Completed"

package/functions/root-data-indexer/index.js

@@ -1,9 +1,8 @@
 /**
  * @fileoverview Root Data Indexer
  * Runs daily to index exactly what data is available for every date.
+ * UPDATED: Includes 'targetDate' optimization for fast single-day updates.
  * UPDATED: Includes explicit checks for Signed-In User Portfolios, History, AND Social Data.
- * FIXED: 'hasSocial' is now strictly for Generic Asset Data. PI/User data is separated.
- * FIXED: PI Collection paths now match the '19M/snapshots' structure.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -63,7 +62,8 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     const { 
         availabilityCollection, 
         earliestDate,
-        collections 
+        collections,
+        targetDate // [NEW] Optional parameter to scan a single specific date
     } = config;
 
     const PRICE_COLLECTION_NAME = 'asset_prices';
@@ -72,45 +72,56 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     const PI_SOCIAL_COLL_NAME      = collections.piSocial || 'pi_social_posts';
     const SIGNED_IN_SOCIAL_COLL_NAME = collections.signedInUserSocialCollection || 'signed_in_users';
 
-    logger.log('INFO', '[RootDataIndexer] Starting Root Data Availability Scan...');
+    const scanMode = targetDate ? 'SINGLE_DATE' : 'FULL_SCAN';
+    logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
     
-    // 1. Price Availability
-    // Sample up to 10 shards to extract date keys (efficient - doesn't read all shards)
+    // 1. Price Availability (Global Scan Mode Only)
+    // If running for a single targetDate, we skip the global shard sampling to save time.
     const priceAvailabilitySet = new Set();
-    try {
-        // Path: asset_prices/shard_*
-        const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
-        const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
-        
-        if (!priceShardsSnapshot.empty) {
-            // Sample up to 10 shards and extract date keys from them
-            // This gives us a representative sample without reading hundreds of shards
-            for (const shardDoc of priceShardsSnapshot.docs) {
-                if (shardDoc.id.startsWith('shard_')) {
-                    const data = shardDoc.data();
-                    Object.values(data).forEach(instrument => { 
-                        if (instrument && instrument.prices) { 
-                            Object.keys(instrument.prices).forEach(dateKey => { 
-                                if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
-                                    priceAvailabilitySet.add(dateKey); 
-                                } 
-                            }); 
-                        } 
-                    });
+    
+    if (!targetDate) {
+        try {
+            // Path: asset_prices/shard_*
+            const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
+            const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
+            
+            if (!priceShardsSnapshot.empty) {
+                // Sample up to 10 shards and extract date keys from them
+                for (const shardDoc of priceShardsSnapshot.docs) {
+                    if (shardDoc.id.startsWith('shard_')) {
+                        const data = shardDoc.data();
+                        Object.values(data).forEach(instrument => { 
+                            if (instrument && instrument.prices) { 
+                                Object.keys(instrument.prices).forEach(dateKey => { 
+                                    if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
+                                        priceAvailabilitySet.add(dateKey); 
+                                    } 
+                                }); 
+                            } 
+                        });
+                    }
                 }
             }
+        } catch (e) {
+            logger.log('ERROR', '[RootDataIndexer] Failed to sample price shards.', { error: e.message });
         }
-    } catch (e) {
-        logger.log('ERROR', '[RootDataIndexer] Failed to sample price shards.', { error: e.message });
     }
 
     // 2. Determine Date Range
-    const start = new Date(earliestDate || '2023-01-01');
-    const end   = new Date();
-    end.setDate(end.getDate() + 1); 
-
     const datesToScan = [];
-    for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) { datesToScan.push(d.toISOString().slice(0, 10)); }
+    if (targetDate) {
+        // [NEW] Single Date Optimization
+        datesToScan.push(targetDate);
+    } else {
+        // [OLD] Full History
+        const start = new Date(earliestDate || '2023-01-01');
+        const end   = new Date();
+        end.setDate(end.getDate() + 1); 
+
+        for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) { 
+            datesToScan.push(d.toISOString().slice(0, 10)); 
+        }
+    }
 
     // 3. Scan in Parallel
     const limit = pLimit(20); 
@@ -211,9 +222,9 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 normHistExists, specHistExists,
                 insightsSnap, socialQuerySnap,
                 piRankingsSnap, 
-                piPortExists,  // UPDATED: Check for any part_* document
-                piDeepExists,  // UPDATED: Check for any part_* document
-                piHistExists,  // UPDATED: Check for any part_* document
+                piPortExists,  
+                piDeepExists, 
+                piHistExists, 
                 signedInPortExists, signedInHistExists, 
                 verificationsQuery,
                 universalSocialSnap
@@ -253,7 +264,6 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.details.speculatorHistory   = specHistExists;
             availability.details.piRankings          = piRankingsSnap.exists;
             
-            // UPDATED: Now checking for any part_* document existence
             availability.details.piPortfolios        = piPortExists; 
             availability.details.piDeepPortfolios    = piDeepExists; 
             availability.details.piHistory           = piHistExists;
@@ -270,17 +280,24 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.details.signedInUserVerification = !verificationsQuery.empty;
 
             // Aggregates
-            // UPDATED: Using part existence checks
             availability.hasPortfolio = normPortExists || specPortExists || piPortExists || signedInPortExists;
             availability.hasHistory   = normHistExists || specHistExists || piHistExists || signedInHistExists;
             availability.hasInsights  = insightsSnap.exists;
-            
-            // [CRITICAL FIX]
-            // hasSocial strictly refers to GENERIC/ASSET social data.
-            // It does NOT include PI or SignedIn data (they have their own flags).
             availability.hasSocial    = !socialQuerySnap.empty; 
             
-            availability.hasPrices    = priceAvailabilitySet.has(dateStr);
+            // Price Check
+            if (targetDate) {
+                // In single-date mode, we do a quick check on the shard if we don't have the full set loaded
+                // For performance, we default hasPrices to false unless we implement a specific single-date shard check.
+                // However, since this is usually called right after fetching, we can leave it as a secondary concern 
+                // or assume the Global Sync will catch it later. 
+                // OPTIONAL: Implementation of specific shard check for single date:
+                // For now, we reuse the set if available (it won't be) or default false.
+                // NOTE: The Global Scan at 01:00 UTC will strictly verify prices.
+                availability.hasPrices = priceAvailabilitySet.has(dateStr); 
+            } else {
+                availability.hasPrices = priceAvailabilitySet.has(dateStr);
+            }
 
             await db.collection(availabilityCollection).doc(dateStr).set(availability);
             updatesCount++;
@@ -288,14 +305,12 @@ exports.runRootDataIndexer = async (config, dependencies) => {
        } catch (e) {
             logger.log('ERROR', `[RootDataIndexer] Failed to index ${dateStr}`, {
                 message: e.message,
-                code: e.code,
-                stack: e.stack,
-                full: JSON.stringify(e, Object.getOwnPropertyNames(e)),
+                code: e.code
             });
         }
     }));
 
     await Promise.all(promises);
-    logger.log('SUCCESS', `[RootDataIndexer] Indexing complete. Updated ${updatesCount} dates.`);
-    return { success: true, count: updatesCount };
+    logger.log('SUCCESS', `[RootDataIndexer] Indexing complete. Updated ${updatesCount} dates. Mode: ${scanMode}`);
+    return { success: true, count: updatesCount, mode: scanMode };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.431",
+  "version": "1.0.433",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [