bulltrackers-module 1.0.279 → 1.0.280

package/functions/computation-system/executors/StandardExecutor.js

@@ -3,6 +3,7 @@
  * UPDATED: Implements Batch Flushing to prevent OOM on large datasets.
  * UPDATED: Removes manual global.gc() calls.
  * UPDATED: Manages incremental sharding states.
+ * UPDATED (IDEA 2): Implemented Computation Profiler (timings).
  */
 const { normalizeName }                                                = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
@@ -10,6 +11,7 @@ const { CachedDataLoader }                                             = require
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
 const mathLayer                                                        = require('../layers/index');
+const { performance }                                                  = require('perf_hooks');
 
 class StandardExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
@@ -53,20 +55,31 @@ class StandardExecutor {
         
         logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
         
-        // Metrics & State Tracking
+        // [IDEA 2] Metrics & State Tracking
         const executionStats = {};
         const shardIndexMap  = {}; // Tracks sharding offsets per calculation
         const aggregatedSuccess = {};
         const aggregatedFailures = [];
 
+        // Initialize Timing Stats per calculation
         Object.keys(state).forEach(name => {
-            executionStats[name] = { processedUsers: 0, skippedUsers: 0 };
+            executionStats[name] = { 
+                processedUsers: 0, 
+                skippedUsers: 0,
+                timings: { setup: 0, stream: 0, processing: 0 } // New
+            };
             shardIndexMap[name]  = 0;
         });
 
+        // [IDEA 2] Measure Setup Time
+        const startSetup = performance.now();
         const cachedLoader = new CachedDataLoader(config, deps);
         await cachedLoader.loadMappings();
+        const setupDuration = performance.now() - startSetup;
         
+        // Distribute setup time
+        Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
+
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -83,13 +96,19 @@ class StandardExecutor {
         let usersSinceLastFlush = 0;
 
         try {
+            // [IDEA 2] Loop wrapper for profiling
             for await (const tP_chunk of tP_iter) {
+                // [IDEA 2] Measure Streaming Time (Gap between processing chunks)
+                const startStream = performance.now();
                 if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
                 if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-                
+                const streamDuration = performance.now() - startStream;
+                Object.keys(executionStats).forEach(name => executionStats[name].timings.stream += streamDuration);
+
                 const chunkSize = Object.keys(tP_chunk).length;
 
-                // Execute chunk for all calcs
+                // [IDEA 2] Measure Processing Time
+                const startProcessing = performance.now();
                 const promises = streamingCalcs.map(calc =>  
                     StandardExecutor.executePerUser(
                         calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, 
@@ -98,6 +117,10 @@ class StandardExecutor {
                     ) 
                 );
                 await Promise.all(promises);
+                const procDuration = performance.now() - startProcessing;
+                
+                // Assign processing time (Note: Parallel execution means total wall time is shared)
+                Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
 
                 usersSinceLastFlush += chunkSize;
 
@@ -161,7 +184,7 @@ class StandardExecutor {
             transformedState[name] = {
                 manifest: inst.manifest,
                 getResult: async () => dataToCommit,
-                _executionStats: executionStats[name] // Attach current stats
+                _executionStats: executionStats[name] // Attach current stats including timings
             };
 
             // ⚠️ CRITICAL: CLEAR MEMORY
@@ -196,6 +219,18 @@ class StandardExecutor {
                     successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
                     successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
                 }
+                
+                // [IDEA 2] Sum timing metrics
+                if (update.metrics?.execution?.timings) {
+                     if (!successAcc[name].metrics.execution) successAcc[name].metrics.execution = { timings: { setup:0, stream:0, processing:0 }};
+                     const tDest = successAcc[name].metrics.execution.timings;
+                     const tSrc  = update.metrics.execution.timings;
+                     
+                     tDest.setup += (tSrc.setup || 0);
+                     tDest.stream += (tSrc.stream || 0);
+                     tDest.processing += (tSrc.processing || 0);
+                }
+
                 // Keep the latest hash/composition info
                 successAcc[name].hash = update.hash;
             }

package/functions/computation-system/persistence/RunRecorder.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Utility for recording computation run attempts (The Audit Logger).
  * UPDATED: Stores 'trigger' reason and 'execution' stats.
+ * UPDATED (IDEA 2): Stores granular timing profiles.
  */
 
 const { FieldValue } = require('../utils/utils'); 
@@ -37,6 +38,10 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
     const anomalies = detailedMetrics.validation?.anomalies || [];
     if (error && error.message && error.message.includes('Data Integrity')) { anomalies.push(error.message); }
 
+    // [IDEA 2] Prepare Execution Stats & Timings
+    const rawExecStats = detailedMetrics.execution || {};
+    const timings = rawExecStats.timings || {};
+
     const runEntry = {
         runId: runId,
         computationName: computation,
@@ -53,8 +58,17 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
             type: (triggerReason && triggerReason.includes('Layer')) ? 'CASCADE' : ((triggerReason && triggerReason.includes('New')) ? 'INIT' : 'UPDATE')
         },
 
-        // [NEW] Execution Stats (Internal Loop Data)
-        executionStats: detailedMetrics.execution || {},
+        // [IDEA 2] Enhanced Execution Stats
+        executionStats: {
+            processedUsers: rawExecStats.processedUsers || 0,
+            skippedUsers:   rawExecStats.skippedUsers || 0,
+            // Explicitly break out timings for BigQuery/Analysis
+            timings: {
+                setupMs:      Math.round(timings.setup || 0),
+                streamMs:     Math.round(timings.stream || 0),
+                processingMs: Math.round(timings.processing || 0)
+            }
+        },
 
         outputStats: {
             sizeMB: sizeMB,
@@ -64,7 +78,7 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
         },
 
         anomalies: anomalies,
-        _schemaVersion: '2.1'
+        _schemaVersion: '2.2' // Bumped for profiler
     };
 
     if (error) {

package/functions/computation-system/tools/BuildReporter.js

@@ -4,6 +4,7 @@
  * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
  * UPDATED: Replaced Batch Writes with Parallel Writes to prevent DEADLINE_EXCEEDED timeouts.
  * FIXED: Ensures 'latest' pointer updates even if detail writes fail.
+ * UPDATED (IDEA 1): Added Dependency Impact Analysis ("Blast Radius").
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
@@ -41,6 +42,34 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
     return false;
 }
 
+/**
+ * Helper: Calculates the transitive closure of dependents (Blast Radius).
+ * Returns the count of direct and total cascading dependents.
+ */
+function calculateBlastRadius(targetCalcName, reverseGraph) {
+    const impactSet = new Set();
+    const queue = [targetCalcName];
+    
+    // BFS Traversal
+    while(queue.length > 0) {
+        const current = queue.shift();
+        const dependents = reverseGraph.get(current) || [];
+        
+        dependents.forEach(child => {
+            if (!impactSet.has(child)) {
+                impactSet.add(child);
+                queue.push(child);
+            }
+        });
+    }
+
+    return {
+        directDependents: (reverseGraph.get(targetCalcName) || []).length,
+        totalCascadingDependents: impactSet.size,
+        affectedCalculations: Array.from(impactSet).slice(0, 50) // Cap list size for storage safety
+    };
+}
+
 /**
  * AUTO-RUN ENTRY POINT
  * Uses transactional locking to prevent race conditions.
@@ -100,6 +129,19 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
+    // [IDEA 1] Build Reverse Dependency Graph (Parent -> Children)
+    const reverseGraph = new Map();
+    manifest.forEach(c => {
+        const parentName = normalizeName(c.name);
+        if (c.dependencies) {
+            c.dependencies.forEach(dep => {
+                const depName = normalizeName(dep);
+                if (!reverseGraph.has(depName)) reverseGraph.set(depName, []);
+                reverseGraph.get(depName).push(parentName);
+            });
+        }
+    });
+
     // Main Report Header
     const reportHeader = {
         buildId,
@@ -169,11 +211,18 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                     return; // EXCLUDED: Date is before data exists
                 }
                 
-                targetArray.push({ 
+                const entry = { 
                     name:   item.name, 
                     reason: item.reason || extraReason,
                     pass:   calcManifest ? calcManifest.pass : '?'
-                });
+                };
+
+                // [IDEA 1] If this is a Re-Run, calculate Blast Radius
+                if (targetArray === dateSummary.rerun) {
+                    entry.impact = calculateBlastRadius(item.name, reverseGraph);
+                }
+
+                targetArray.push(entry);
             };
 
             // 1. RUN (New)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.279",
+  "version": "1.0.280",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [