bulltrackers-module 1.0.295 → 1.0.297

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,10 +1,6 @@
 /**
  * @fileoverview Executor for "Standard" (per-user) calculations.
- * UPDATED: Implements Batch Flushing to prevent OOM on large datasets.
- * UPDATED: Implements "Circuit Breaker" to fail fast on high error rates.
- * UPDATED: Implements "Adaptive Flushing" based on V8 Heap usage.
- * UPDATED: Manages incremental sharding states.
- * UPDATED: Implements 'isInitialWrite' flag for robust cleanup.
+ * UPDATED: Tracks IO Operations (Reads/Writes) for Cost Analysis.
  */
 const { normalizeName }                                                = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
@@ -20,7 +16,7 @@ class StandardExecutor {
         const dStr   = date.toISOString().slice(0, 10);
         const logger = deps.logger;
         
-        // 1. Prepare Yesterdays Data if needed
+        // 1. Prepare Yesterdays Data (Counts as Read Ops)
         const fullRoot = { ...rootData };
         if (calcs.some(c => c.isHistorical)) {
             const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
@@ -28,22 +24,17 @@ class StandardExecutor {
             fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
         }
 
-        // 2. Initialize Instances
         const state = {};
         for (const c of calcs) { 
             try { 
                 const inst = new c.class(); 
                 inst.manifest = c; 
-                // Ensure internal storage exists for flushing
                 inst.results = {}; 
                 state[normalizeName(c.name)] = inst; 
                 logger.log('INFO', `${c.name} calculation running for ${dStr}`); 
-            } catch (e) { 
-                logger.log('WARN', `Failed to init ${c.name}`); 
-            } 
+            } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        // 3. Stream, Process & Batch Flush
         return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite);
     }
 
@@ -54,33 +45,34 @@ class StandardExecutor {
         
         if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
         
-        logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
-        
+        // [NEW] Calculate Total Read Ops for this execution context
+        // Each reference in the arrays corresponds to a document fetch
+        let totalReadOps = (portfolioRefs?.length || 0) + (historyRefs?.length || 0);
+        if (rootData.yesterdayPortfolioRefs) totalReadOps += rootData.yesterdayPortfolioRefs.length;
+        // Add +2 for Insights & Social (1 doc each)
+        totalReadOps += 2; 
+
+        // Distribute read costs evenly among calculations (approximation)
+        const readOpsPerCalc = Math.ceil(totalReadOps / streamingCalcs.length);
+
         const executionStats = {};
         const shardIndexMap  = {}; 
         const aggregatedSuccess = {};
         const aggregatedFailures = [];
-
-        // [NEW] Global Error Tracking for Circuit Breaker
         const errorStats = { count: 0, total: 0 };
 
         Object.keys(state).forEach(name => {
             executionStats[name] = { 
-                processedUsers: 0, 
-                skippedUsers: 0,
-                timings: { setup: 0, stream: 0, processing: 0 }
+                processedUsers: 0, skippedUsers: 0, timings: { setup: 0, stream: 0, processing: 0 }
             };
-            shardIndexMap[name]  = 0;
+            shardIndexMap[name] = 0;
         });
 
-        // Track if we have performed a flush yet (for cleanup logic)
         let hasFlushed = false;
-
-        const startSetup = performance.now();
         const cachedLoader = new CachedDataLoader(config, deps);
+        const startSetup = performance.now();
         await cachedLoader.loadMappings();
         const setupDuration = performance.now() - startSetup;
-        
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
@@ -93,8 +85,6 @@ class StandardExecutor {
         const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
         let yP_chunk = {}, tH_chunk = {};
-        
-        const MIN_BATCH_SIZE = 1000; // Minimum to process before checking stats
         let usersSinceLastFlush = 0;
 
         try {
@@ -106,52 +96,26 @@ class StandardExecutor {
                 Object.keys(executionStats).forEach(name => executionStats[name].timings.stream += streamDuration);
 
                 const chunkSize = Object.keys(tP_chunk).length;
-
                 const startProcessing = performance.now();
                 
-                // [UPDATED] Collect execution results (success/failure counts)
-                const promises = streamingCalcs.map(calc =>  
+                const batchResults = await Promise.all(streamingCalcs.map(calc =>  
                     StandardExecutor.executePerUser(
                         calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, 
                         fetchedDeps, previousFetchedDeps, config, deps, cachedLoader,
                         executionStats[normalizeName(calc.manifest.name)]
                     ) 
-                );
+                ));
                 
-                const batchResults = await Promise.all(promises);
                 const procDuration = performance.now() - startProcessing;
-                
                 Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
 
-                // [NEW] Update Error Stats
-                batchResults.forEach(r => {
-                    errorStats.total += (r.success + r.failures);
-                    errorStats.count += r.failures;
-                });
-
-                // [NEW] Circuit Breaker: Fail fast if error rate > 10% after processing 100+ items
-                // We check total > 100 to avoid failing on the very first user if they happen to be bad.
-                if (errorStats.total > 100 && (errorStats.count / errorStats.total) > 0.10) {
-                    const failRate = (errorStats.count / errorStats.total * 100).toFixed(1);
-                    throw new Error(`[Circuit Breaker] High failure rate detected (${failRate}%). Aborting batch to prevent silent data loss.`);
-                }
+                batchResults.forEach(r => { errorStats.total += (r.success + r.failures); errorStats.count += r.failures; });
+                if (errorStats.total > 100 && (errorStats.count / errorStats.total) > 0.10) { throw new Error(`[Circuit Breaker] High failure rate detected.`); }
 
                 usersSinceLastFlush += chunkSize;
-
-                // [NEW] Adaptive Flushing (Memory Pressure Check)
                 const heapStats = v8.getHeapStatistics();
-                const heapUsedRatio = heapStats.used_heap_size / heapStats.heap_size_limit;
-                const MEMORY_THRESHOLD = 0.70; // 70% of available RAM
-                const COUNT_THRESHOLD = 5000;
-
-                if (usersSinceLastFlush >= COUNT_THRESHOLD || heapUsedRatio > MEMORY_THRESHOLD) {
-                    const reason = heapUsedRatio > MEMORY_THRESHOLD ? `MEMORY_PRESSURE (${(heapUsedRatio*100).toFixed(0)}%)` : 'BATCH_LIMIT';
-                    
-                    logger.log('INFO', `[${passName}] 🛁 Flushing buffer after ${usersSinceLastFlush} users. Reason: ${reason}`);
-                    
-                    // [UPDATED] Pass isInitialWrite: true only on the first flush
+                if (usersSinceLastFlush >= 5000 || (heapStats.used_heap_size / heapStats.heap_size_limit) > 0.70) {
                     const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
-                    
                     hasFlushed = true;
                     StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);
                     usersSinceLastFlush = 0;
@@ -161,22 +125,23 @@ class StandardExecutor {
             if (yP_iter && yP_iter.return) await yP_iter.return();
             if (tH_iter && tH_iter.return) await tH_iter.return();
         }
-        
-        logger.log('INFO', `[${passName}] Streaming complete. Performing final commit.`);
-        // [UPDATED] If we never flushed in the loop, this is the initial write
+
         const finalResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'FINAL', skipStatusWrite, !hasFlushed);
-        
         StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, finalResult);
 
+        // [NEW] Inject Read Ops into the final report
+        Object.values(aggregatedSuccess).forEach(update => {
+            if (!update.metrics.io) update.metrics.io = { reads: 0, writes: 0, deletes: 0 };
+            update.metrics.io.reads = readOpsPerCalc;
+        });
+
         return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
     }
 
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
         const transformedState = {};
-        
         for (const [name, inst] of Object.entries(state)) {
             const rawResult = inst.results || {};
-            
             const firstUser = Object.keys(rawResult)[0];
             let dataToCommit = rawResult;
 
@@ -199,43 +164,40 @@ class StandardExecutor {
                 getResult: async () => dataToCommit,
                 _executionStats: executionStats[name]
             };
-
-            // Clear the memory immediately after preparing the commit
             inst.results = {};
         }
 
-        // [UPDATED] Pass isInitialWrite to ResultCommitter
         const result = await commitResults(transformedState, dateStr, passName, config, deps, skipStatusWrite, {
-            flushMode: mode,      
-            shardIndexes: shardIndexMap,
-            isInitialWrite: isInitialWrite 
+            flushMode: mode, shardIndexes: shardIndexMap, isInitialWrite: isInitialWrite 
         });
 
-        if (result.shardIndexes) {
-            Object.assign(shardIndexMap, result.shardIndexes);
-        }
-
+        if (result.shardIndexes) Object.assign(shardIndexMap, result.shardIndexes);
         return result;
     }
 
     static mergeReports(successAcc, failureAcc, newResult) {
         if (!newResult) return;
-        
         for (const [name, update] of Object.entries(newResult.successUpdates)) {
             if (!successAcc[name]) {
                 successAcc[name] = update;
             } else {
-                if (update.metrics?.storage) {
-                    successAcc[name].metrics.storage.sizeBytes += (update.metrics.storage.sizeBytes || 0);
-                    successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
-                    successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
-                }
+                // Merge Storage metrics
+                successAcc[name].metrics.storage.sizeBytes += (update.metrics.storage.sizeBytes || 0);
+                successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
+                successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
                 
+                // [NEW] Merge IO Metrics
+                if (update.metrics.io) {
+                    if (!successAcc[name].metrics.io) successAcc[name].metrics.io = { writes: 0, deletes: 0, reads: 0 };
+                    successAcc[name].metrics.io.writes += (update.metrics.io.writes || 0);
+                    successAcc[name].metrics.io.deletes += (update.metrics.io.deletes || 0);
+                }
+
+                // Merge timings
                 if (update.metrics?.execution?.timings) {
                      if (!successAcc[name].metrics.execution) successAcc[name].metrics.execution = { timings: { setup:0, stream:0, processing:0 }};
                      const tDest = successAcc[name].metrics.execution.timings;
                      const tSrc  = update.metrics.execution.timings;
-                     
                      tDest.setup += (tSrc.setup || 0);
                      tDest.stream += (tSrc.stream || 0);
                      tDest.processing += (tSrc.processing || 0);
@@ -243,12 +205,9 @@ class StandardExecutor {
                 successAcc[name].hash = update.hash;
             }
         }
-
-        if (newResult.failureReport) {
-            failureAcc.push(...newResult.failureReport);
-        }
+        if (newResult.failureReport) failureAcc.push(...newResult.failureReport);
     }
-
+    
     static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats) {
         const { logger } = deps;
         const targetUserType = metadata.userType; 
@@ -256,7 +215,6 @@ class StandardExecutor {
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const SCHEMAS  = mathLayer.SCHEMAS;
 
-        // [NEW] Track local batch success/failure
         let chunkSuccess = 0;
         let chunkFailures = 0;
 

package/functions/computation-system/helpers/computation_worker.js

@@ -2,7 +2,7 @@
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
  * UPDATED: Implements IAM Auth for Workflow Callbacks.
- * UPDATED: Implements Memory Heartbeat (Flight Recorder) for OOM detection.
+ * UPDATED: Implements Peak Memory Heartbeat and Resource Tier tracking.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -11,6 +11,7 @@ const { StructuredLogger }    = require('../logger/logger');
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
 const https                   = require('https'); 
 const { GoogleAuth }          = require('google-auth-library'); 
+const { normalizeName }       = require('../utils/utils');
 
 let calculationPackage;
 try { calculationPackage = require('aiden-shared-calculations-unified');
@@ -20,15 +21,19 @@ const calculations = calculationPackage.calculations;
 const MAX_RETRIES = 3;
 
 /**
- * [NEW] Helper: Starts a background heartbeat to track memory usage.
- * This acts as a "Black Box Recorder". If the worker crashes (OOM),
- * the last written value will remain in Firestore for the Dispatcher to analyze.
+ * [UPDATED] Heartbeat now returns a closure to get the PEAK memory.
+ * This acts as a "Black Box Recorder".
  */
 function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
+    let peakRss = 0;
+    
     const getMemStats = () => {
         const mem = process.memoryUsage();
+        const rssMB = Math.round(mem.rss / 1024 / 1024);
+        if (rssMB > peakRss) peakRss = rssMB;
+        
         return {
-            rssMB: Math.round(mem.rss / 1024 / 1024),       // Resident Set Size (OOM Killer Metric)
+            rssMB: rssMB,
             heapUsedMB: Math.round(mem.heapUsed / 1024 / 1024),
             timestamp: new Date()
         };
@@ -50,7 +55,10 @@ function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
     // Unref so this timer doesn't prevent the process from exiting naturally
     timer.unref();
 
-    return timer;
+    return {
+        timer,
+        getPeak: () => peakRss
+    };
 }
 
 /**
@@ -127,7 +135,9 @@ async function handleComputationTask(message, config, dependencies) {
 
     if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
     
-    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, metaStatePath } = data;
+    // [UPDATED] Extract 'resources' from payload (set by Dispatcher)
+    const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, metaStatePath, resources } = data;
+    const resourceTier = resources || 'standard'; // Default to standard
     
     if (!date || !pass || !computation) { logger.log('ERROR', `[Worker] Invalid payload.`, data); return; }
 
@@ -158,7 +168,7 @@ async function handleComputationTask(message, config, dependencies) {
             } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
     }
 
-    logger.log('INFO', `[Worker] 📥 Received Task: ${computation} (${date}) [Attempt ${retryCount}/${MAX_RETRIES}]`);
+    logger.log('INFO', `[Worker] 📥 Received Task: ${computation} (${date}) [Attempt ${retryCount}/${MAX_RETRIES}] [Tier: ${resourceTier}]`);
 
     // 1. Update Status to IN_PROGRESS & Initialize Telemetry
     try {
@@ -172,12 +182,13 @@ async function handleComputationTask(message, config, dependencies) {
     } catch (leaseErr) {}
 
     // 2. START HEARTBEAT (The Flight Recorder)
-    const heartbeatTimer = startMemoryHeartbeat(db, ledgerPath, 2000);
+    // [UPDATED] Using new logic to track peak
+    const heartbeatControl = startMemoryHeartbeat(db, ledgerPath, 2000);
 
     let computationManifest;
     try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) { 
-        clearInterval(heartbeatTimer); // Stop if we fail early
+        clearInterval(heartbeatControl.timer); // Stop if we fail early
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
         return; 
     }
@@ -191,7 +202,7 @@ async function handleComputationTask(message, config, dependencies) {
         const duration = Date.now() - startTime;
 
         // STOP HEARTBEAT ON SUCCESS
-        clearInterval(heartbeatTimer);
+        clearInterval(heartbeatControl.timer);
 
         const failureReport  = result?.updates?.failureReport  || [];
         const successUpdates = result?.updates?.successUpdates || {};
@@ -203,20 +214,33 @@ async function handleComputationTask(message, config, dependencies) {
         else {
             if (Object.keys(successUpdates).length > 0) { logger.log('INFO', `[Worker] ✅ Stored: ${computation}`); } 
             else { logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`); }
+            
+            // Extract the metrics from the success update for the recorder
+            const calcUpdate = successUpdates[normalizeName(computation)] || {};
+            const finalMetrics = {
+                durationMs: duration,
+                peakMemoryMB: heartbeatControl.getPeak(),
+                io: calcUpdate.metrics?.io,
+                storage: calcUpdate.metrics?.storage,
+                execution: calcUpdate.metrics?.execution,
+                validation: calcUpdate.metrics?.validation,
+                composition: calcUpdate.composition
+            };
 
             await db.doc(ledgerPath).update({
                 status: 'COMPLETED',
                 completedAt: new Date()
             }).catch(() => {});
 
-            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration }, triggerReason);
+            // [UPDATED] Pass resourceTier and metrics to recordRunAttempt
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, finalMetrics, triggerReason, resourceTier);
 
             const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
             if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
         }
     } catch (err) { 
         // STOP HEARTBEAT ON ERROR
-        clearInterval(heartbeatTimer);
+        clearInterval(heartbeatControl.timer);
 
         // --- ERROR HANDLING ---
         const isDeterministicError = err.stage === 'SHARDING_LIMIT_EXCEEDED' || 
@@ -241,7 +265,7 @@ async function handleComputationTask(message, config, dependencies) {
                     failedAt: new Date()
                 }, { merge: true });
 
-                 await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0 }, triggerReason);
+                 await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0, peakMemoryMB: heartbeatControl.getPeak() }, triggerReason, resourceTier);
                  
                  const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
                  if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
@@ -252,9 +276,9 @@ async function handleComputationTask(message, config, dependencies) {
         if (retryCount >= MAX_RETRIES) { throw err; }
 
         logger.log('ERROR', `[Worker] ❌ Crash: ${computation}: ${err.message}`); 
-        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0 }, triggerReason);
+        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0, peakMemoryMB: heartbeatControl.getPeak() }, triggerReason, resourceTier);
         throw err; 
     }
 }
 
-module.exports = { handleComputationTask };
+module.exports = { handleComputationTask };