bulltrackers-module 1.0.278 → 1.0.280

package/functions/computation-system/executors/StandardExecutor.js

@@ -3,6 +3,7 @@
  * UPDATED: Implements Batch Flushing to prevent OOM on large datasets.
  * UPDATED: Removes manual global.gc() calls.
  * UPDATED: Manages incremental sharding states.
+ * UPDATED (IDEA 2): Implemented Computation Profiler (timings).
  */
 const { normalizeName }                                                = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
@@ -10,6 +11,7 @@ const { CachedDataLoader }                                             = require
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
 const mathLayer                                                        = require('../layers/index');
+const { performance }                                                  = require('perf_hooks');
 
 class StandardExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
@@ -53,20 +55,31 @@ class StandardExecutor {
         
         logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
         
-        // Metrics & State Tracking
+        // [IDEA 2] Metrics & State Tracking
         const executionStats = {};
         const shardIndexMap  = {}; // Tracks sharding offsets per calculation
         const aggregatedSuccess = {};
         const aggregatedFailures = [];
 
+        // Initialize Timing Stats per calculation
         Object.keys(state).forEach(name => {
-            executionStats[name] = { processedUsers: 0, skippedUsers: 0 };
+            executionStats[name] = { 
+                processedUsers: 0, 
+                skippedUsers: 0,
+                timings: { setup: 0, stream: 0, processing: 0 } // New
+            };
             shardIndexMap[name]  = 0;
         });
 
+        // [IDEA 2] Measure Setup Time
+        const startSetup = performance.now();
         const cachedLoader = new CachedDataLoader(config, deps);
         await cachedLoader.loadMappings();
+        const setupDuration = performance.now() - startSetup;
         
+        // Distribute setup time
+        Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
+
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -83,13 +96,19 @@ class StandardExecutor {
         let usersSinceLastFlush = 0;
 
         try {
+            // [IDEA 2] Loop wrapper for profiling
             for await (const tP_chunk of tP_iter) {
+                // [IDEA 2] Measure Streaming Time (Gap between processing chunks)
+                const startStream = performance.now();
                 if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
                 if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-                
+                const streamDuration = performance.now() - startStream;
+                Object.keys(executionStats).forEach(name => executionStats[name].timings.stream += streamDuration);
+
                 const chunkSize = Object.keys(tP_chunk).length;
 
-                // Execute chunk for all calcs
+                // [IDEA 2] Measure Processing Time
+                const startProcessing = performance.now();
                 const promises = streamingCalcs.map(calc =>  
                     StandardExecutor.executePerUser(
                         calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, 
@@ -98,6 +117,10 @@ class StandardExecutor {
                     ) 
                 );
                 await Promise.all(promises);
+                const procDuration = performance.now() - startProcessing;
+                
+                // Assign processing time (Note: Parallel execution means total wall time is shared)
+                Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
 
                 usersSinceLastFlush += chunkSize;
 
@@ -161,7 +184,7 @@ class StandardExecutor {
             transformedState[name] = {
                 manifest: inst.manifest,
                 getResult: async () => dataToCommit,
-                _executionStats: executionStats[name] // Attach current stats
+                _executionStats: executionStats[name] // Attach current stats including timings
             };
 
             // ⚠️ CRITICAL: CLEAR MEMORY
@@ -196,6 +219,18 @@ class StandardExecutor {
                     successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
                     successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
                 }
+                
+                // [IDEA 2] Sum timing metrics
+                if (update.metrics?.execution?.timings) {
+                     if (!successAcc[name].metrics.execution) successAcc[name].metrics.execution = { timings: { setup:0, stream:0, processing:0 }};
+                     const tDest = successAcc[name].metrics.execution.timings;
+                     const tSrc  = update.metrics.execution.timings;
+                     
+                     tDest.setup += (tSrc.setup || 0);
+                     tDest.stream += (tSrc.stream || 0);
+                     tDest.processing += (tSrc.processing || 0);
+                }
+
                 // Keep the latest hash/composition info
                 successAcc[name].hash = update.hash;
             }

package/functions/computation-system/persistence/ResultCommitter.js

@@ -11,7 +11,7 @@ const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
 const { HeuristicValidator }               = require('./ResultsValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
-const zlib                                 = require('zlib'); // [NEW] Compression Lib
+const zlib                                 = require('zlib'); 
 
 const NON_RETRYABLE_ERRORS = [
     'PERMISSION_DENIED',     'DATA_LOSS',             'FAILED_PRECONDITION'
@@ -59,10 +59,11 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             let effectiveOverrides = { ...configOverrides };
             
             if (isPriceOnly && !effectiveOverrides.weekend) {
+                // Apply strict leniency for weekend/holiday price actions
                 effectiveOverrides.weekend = {
                     maxZeroPct: 100,
                     maxFlatlinePct: 100,
-                    maxNullPct: 100 // Allow full nulls (e.g. holidays)
+                    maxNullPct: 100 
                 };
             }
             // -----------------------------------------------------------
@@ -193,17 +194,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD') {
     
-    // --- [NEW] COMPRESSION STRATEGY ---
-    // Try to compress before falling back to complex sharding
+    // --- COMPRESSION STRATEGY ---
     try {
         const jsonString = JSON.stringify(result);
         const rawBuffer = Buffer.from(jsonString);
 
-        // Only attempt if meaningful size (> 50KB)
         if (rawBuffer.length > 50 * 1024) { 
             const compressedBuffer = zlib.gzipSync(rawBuffer);
             
-            // If compressed fits in one document (< 900KB safety limit)
             if (compressedBuffer.length < 900 * 1024) {
                 logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB. Saved as Blob.`);
                 
@@ -214,7 +212,6 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
                     payload: compressedBuffer
                 };
                 
-                // Write immediately
                 await docRef.set(compressedPayload, { merge: true });
                 
                 return { 

package/functions/computation-system/persistence/RunRecorder.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Utility for recording computation run attempts (The Audit Logger).
  * UPDATED: Stores 'trigger' reason and 'execution' stats.
+ * UPDATED (IDEA 2): Stores granular timing profiles.
  */
 
 const { FieldValue } = require('../utils/utils'); 
@@ -37,6 +38,10 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
     const anomalies = detailedMetrics.validation?.anomalies || [];
     if (error && error.message && error.message.includes('Data Integrity')) { anomalies.push(error.message); }
 
+    // [IDEA 2] Prepare Execution Stats & Timings
+    const rawExecStats = detailedMetrics.execution || {};
+    const timings = rawExecStats.timings || {};
+
     const runEntry = {
         runId: runId,
         computationName: computation,
@@ -53,8 +58,17 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
             type: (triggerReason && triggerReason.includes('Layer')) ? 'CASCADE' : ((triggerReason && triggerReason.includes('New')) ? 'INIT' : 'UPDATE')
         },
 
-        // [NEW] Execution Stats (Internal Loop Data)
-        executionStats: detailedMetrics.execution || {},
+        // [IDEA 2] Enhanced Execution Stats
+        executionStats: {
+            processedUsers: rawExecStats.processedUsers || 0,
+            skippedUsers:   rawExecStats.skippedUsers || 0,
+            // Explicitly break out timings for BigQuery/Analysis
+            timings: {
+                setupMs:      Math.round(timings.setup || 0),
+                streamMs:     Math.round(timings.stream || 0),
+                processingMs: Math.round(timings.processing || 0)
+            }
+        },
 
         outputStats: {
             sizeMB: sizeMB,
@@ -64,7 +78,7 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
         },
 
         anomalies: anomalies,
-        _schemaVersion: '2.1'
+        _schemaVersion: '2.2' // Bumped for profiler
     };
 
     if (error) {

package/functions/computation-system/tools/BuildReporter.js

@@ -2,16 +2,15 @@
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
  * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
- * UPDATED: Added transactional locking to prevent duplicate reports on concurrent cold starts.
- * UPDATED: Adds 'pass' number to detail records for better waterfall visibility.
+ * UPDATED: Replaced Batch Writes with Parallel Writes to prevent DEADLINE_EXCEEDED timeouts.
  * FIXED: Ensures 'latest' pointer updates even if detail writes fail.
+ * UPDATED (IDEA 1): Added Dependency Impact Analysis ("Blast Radius").
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
 const { fetchComputationStatus }                                           = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability }                                        = require('../data/AvailabilityChecker');
-const { commitBatchInChunks }                                              = require('../persistence/FirestoreUtils');
 const pLimit                                                               = require('p-limit');
 const path                                                                 = require('path');
 const packageJson                                                          = require(path.join(__dirname, '..', '..', '..', 'package.json'));
@@ -43,10 +42,37 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
     return false;
 }
 
+/**
+ * Helper: Calculates the transitive closure of dependents (Blast Radius).
+ * Returns the count of direct and total cascading dependents.
+ */
+function calculateBlastRadius(targetCalcName, reverseGraph) {
+    const impactSet = new Set();
+    const queue = [targetCalcName];
+    
+    // BFS Traversal
+    while(queue.length > 0) {
+        const current = queue.shift();
+        const dependents = reverseGraph.get(current) || [];
+        
+        dependents.forEach(child => {
+            if (!impactSet.has(child)) {
+                impactSet.add(child);
+                queue.push(child);
+            }
+        });
+    }
+
+    return {
+        directDependents: (reverseGraph.get(targetCalcName) || []).length,
+        totalCascadingDependents: impactSet.size,
+        affectedCalculations: Array.from(impactSet).slice(0, 50) // Cap list size for storage safety
+    };
+}
+
 /**
  * AUTO-RUN ENTRY POINT
- * UPDATED: Uses transactional locking to prevent race conditions.
- * If we deploy multiple computation pass nodes simultaneously, only one should run the report.
+ * Uses transactional locking to prevent race conditions.
  */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
@@ -88,7 +114,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
 }
 
 /**
- * Generates the report and saves to Firestore (Sharded).
+ * Generates the report and saves to Firestore.
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
@@ -103,6 +129,19 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
+    // [IDEA 1] Build Reverse Dependency Graph (Parent -> Children)
+    const reverseGraph = new Map();
+    manifest.forEach(c => {
+        const parentName = normalizeName(c.name);
+        if (c.dependencies) {
+            c.dependencies.forEach(dep => {
+                const depName = normalizeName(dep);
+                if (!reverseGraph.has(depName)) reverseGraph.set(depName, []);
+                reverseGraph.get(depName).push(parentName);
+            });
+        }
+    });
+
     // Main Report Header
     const reportHeader = {
         buildId,
@@ -153,7 +192,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 impossible: [], // Missing Data (Historical) / Impossible Dependency
                 uptodate:   [], // Hash Match (Previously "Skipped")
                 
-                // [NEW] Metadata for Verification
+                // Metadata for Verification
                 meta: {
                     totalIncluded: 0,
                     totalExpected: 0,
@@ -166,18 +205,24 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             dateSummary.meta.totalExpected = expectedCount;
 
             // Helper to push only if date is valid for this specific calc
-            // [UPDATED] Adds 'pass' number to the record
             const pushIfValid = (targetArray, item, extraReason = null) => {
                 const calcManifest = manifestMap.get(item.name);
                 if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) {
                     return; // EXCLUDED: Date is before data exists
                 }
                 
-                targetArray.push({ 
+                const entry = { 
                     name:   item.name, 
                     reason: item.reason || extraReason,
                     pass:   calcManifest ? calcManifest.pass : '?'
-                });
+                };
+
+                // [IDEA 1] If this is a Re-Run, calculate Blast Radius
+                if (targetArray === dateSummary.rerun) {
+                    entry.impact = calculateBlastRadius(item.name, reverseGraph);
+                }
+
+                targetArray.push(entry);
             };
 
             // 1. RUN (New)
@@ -187,7 +232,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             analysis.reRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Hash Mismatch"));
 
             // 3. BLOCKED (Temporary Issues)
-            // Merging 'blocked' and 'failedDependency' as both are temporary blocks
             analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
             analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
 
@@ -211,7 +255,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 logger.log('WARN', `[BuildReporter] ⚠️ Mismatch on ${dateStr}: Expected ${expectedCount} but got ${includedCount}.`);
             }
 
-            // ALWAYS WRITE THE REPORT (No filtering based on activity)
+            // QUEUE THE WRITE (Don't write yet)
             const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
             detailWrites.push({
                 ref: detailRef,
@@ -247,22 +291,33 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportHeader);
 
-    // 2. Write Details (Protected)
-    // [FIX] We wrap this in try-catch so that if the massive detail write fails,
-    // we still update the 'latest' pointer to the new version.
+    // 2. Write Details (Protected & Parallelized)
+    // FIX: Using parallel individual writes instead of Batch to avoid DEADLINE_EXCEEDED
     let detailsSuccess = true;
     if (detailWrites.length > 0) {
-        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records...`);
+        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records (Parallel Strategy)...`);
+        
         try {
-            await commitBatchInChunks(config, dependencies, detailWrites, 'BuildReportDetails');
+            // Concurrency limit of 15 to be safe
+            const writeLimit = pLimit(15);
+            const writePromises = detailWrites.map(w => writeLimit(() => 
+                w.ref.set(w.data).catch(e => {
+                    logger.log('WARN', `[BuildReporter] Failed to write detail for ${w.ref.path}: ${e.message}`);
+                    throw e;
+                })
+            ));
+            
+            await Promise.all(writePromises);
+            logger.log('INFO', `[BuildReporter] Successfully wrote all detail records.`);
+
         } catch (detailErr) {
             detailsSuccess = false;
-            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write all details, but Report Header is saved.`, detailErr);
+            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write some details. Report Header is preserved.`, detailErr);
         }
     }
 
     // 3. Update 'latest' pointer
-    // This now runs even if details failed, preventing the version mismatch bug.
+    // This runs regardless of detail write success/failure
     const latestMetadata = { 
         ...reportHeader, 
         note: detailsSuccess 
@@ -270,9 +325,12 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             : "Latest build report pointer (WARNING: Partial detail records due to write error)."
     };
     
-    await db.collection('computation_build_records').doc('latest').set(latestMetadata);
-
-    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}.`);
+    try {
+        await db.collection('computation_build_records').doc('latest').set(latestMetadata);
+        logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}. Pointer Updated.`);
+    } catch (pointerErr) {
+        logger.log('FATAL', `[BuildReporter] Failed to update 'latest' pointer!`, pointerErr);
+    }
     
     return {
         success: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.278",
+  "version": "1.0.280",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [