bulltrackers-module 1.0.280 → 1.0.282

package/functions/computation-system/executors/MetaExecutor.js

@@ -2,7 +2,7 @@
  * @fileoverview Executor for "Meta" (global) calculations.
  * UPDATED: Uses CachedDataLoader for all data access.
  * UPDATED: Tracks processed shard/item counts.
- * UPDATED: Removed global.gc() calls.
+ * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
  */
 const { normalizeName }    = require('../utils/utils');
 const { CachedDataLoader } = require('../data/CachedDataLoader');
@@ -27,7 +27,9 @@ class MetaExecutor {
                 deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); 
             }
         }
-        return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
+        
+        // [UPDATED] Meta Calcs run once per day, so isInitialWrite is always true
+        return await commitResults(state, dStr, passName, config, deps, skipStatusWrite, { isInitialWrite: true });
     }
 
     static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
@@ -35,7 +37,6 @@ class MetaExecutor {
         const { logger } = deps;
         const stats = { processedShards: 0, processedItems: 0 };
         
-        // Lazy fetch insights/social using the loader
         const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
         
@@ -59,12 +60,9 @@ class MetaExecutor {
                 
                 stats.processedShards++;
                 stats.processedItems += Object.keys(shardData).length;
-
-                // Removed global.gc()
             }
             logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
             
-            // Attach stats
             calcInstance._executionStats = stats;
             return calcInstance.getResult ? await calcInstance.getResult() : {};
         } else {
@@ -75,7 +73,7 @@ class MetaExecutor {
             });
             const res = await calcInstance.process(context);
             
-            stats.processedItems = 1; // "Global" item
+            stats.processedItems = 1; 
             calcInstance._executionStats = stats;
             
             return res;
@@ -83,4 +81,4 @@ class MetaExecutor {
     }
 }
 
-module.exports = { MetaExecutor };
+module.exports = { MetaExecutor };

package/functions/computation-system/executors/StandardExecutor.js

@@ -3,7 +3,7 @@
  * UPDATED: Implements Batch Flushing to prevent OOM on large datasets.
  * UPDATED: Removes manual global.gc() calls.
  * UPDATED: Manages incremental sharding states.
- * UPDATED (IDEA 2): Implemented Computation Profiler (timings).
+ * UPDATED: Implements 'isInitialWrite' flag for robust cleanup.
  */
 const { normalizeName }                                                = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
@@ -42,7 +42,6 @@ class StandardExecutor {
         }
 
         // 3. Stream, Process & Batch Flush
-        // The return value contains the aggregated success/failure reports from all flushes
         return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite);
     }
 
@@ -55,29 +54,28 @@ class StandardExecutor {
         
         logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
         
-        // [IDEA 2] Metrics & State Tracking
         const executionStats = {};
-        const shardIndexMap  = {}; // Tracks sharding offsets per calculation
+        const shardIndexMap  = {}; 
         const aggregatedSuccess = {};
         const aggregatedFailures = [];
 
-        // Initialize Timing Stats per calculation
         Object.keys(state).forEach(name => {
             executionStats[name] = { 
                 processedUsers: 0, 
                 skippedUsers: 0,
-                timings: { setup: 0, stream: 0, processing: 0 } // New
+                timings: { setup: 0, stream: 0, processing: 0 }
             };
             shardIndexMap[name]  = 0;
         });
 
-        // [IDEA 2] Measure Setup Time
+        // Track if we have performed a flush yet (for cleanup logic)
+        let hasFlushed = false;
+
         const startSetup = performance.now();
         const cachedLoader = new CachedDataLoader(config, deps);
         await cachedLoader.loadMappings();
         const setupDuration = performance.now() - startSetup;
         
-        // Distribute setup time
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
@@ -91,14 +89,11 @@ class StandardExecutor {
 
         let yP_chunk = {}, tH_chunk = {};
         
-        // OOM Protection: Batch Flushing Configuration
-        const BATCH_SIZE = 5000; // Flush every 5000 users
+        const BATCH_SIZE = 5000;
         let usersSinceLastFlush = 0;
 
         try {
-            // [IDEA 2] Loop wrapper for profiling
             for await (const tP_chunk of tP_iter) {
-                // [IDEA 2] Measure Streaming Time (Gap between processing chunks)
                 const startStream = performance.now();
                 if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
                 if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
@@ -107,7 +102,6 @@ class StandardExecutor {
 
                 const chunkSize = Object.keys(tP_chunk).length;
 
-                // [IDEA 2] Measure Processing Time
                 const startProcessing = performance.now();
                 const promises = streamingCalcs.map(calc =>  
                     StandardExecutor.executePerUser(
@@ -119,20 +113,18 @@ class StandardExecutor {
                 await Promise.all(promises);
                 const procDuration = performance.now() - startProcessing;
                 
-                // Assign processing time (Note: Parallel execution means total wall time is shared)
                 Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
 
                 usersSinceLastFlush += chunkSize;
 
-                // --- BATCH FLUSH CHECK ---
                 if (usersSinceLastFlush >= BATCH_SIZE) {
                     logger.log('INFO', `[${passName}] 🛁 Flushing buffer after ${usersSinceLastFlush} users...`);
                     
-                    const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true);
+                    // [UPDATED] Pass isInitialWrite: true only on the first flush
+                    const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
                     
-                    // Aggregate metrics
+                    hasFlushed = true;
                     StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, flushResult);
-                    
                     usersSinceLastFlush = 0;
                 }
             }
@@ -141,34 +133,27 @@ class StandardExecutor {
             if (tH_iter && tH_iter.return) await tH_iter.return();
         }
         
-        // --- FINAL FLUSH ---
         logger.log('INFO', `[${passName}] Streaming complete. Performing final commit.`);
-        const finalResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'FINAL', skipStatusWrite);
+        // [UPDATED] If we never flushed in the loop, this is the initial write
+        const finalResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'FINAL', skipStatusWrite, !hasFlushed);
         
         StandardExecutor.mergeReports(aggregatedSuccess, aggregatedFailures, finalResult);
 
         return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
     }
 
-    static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite) {
+    static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
         const transformedState = {};
-        const { logger } = deps;
-
-        // 1. Prepare and Clear Instances
+        
         for (const [name, inst] of Object.entries(state)) {
-            // Get data from the standard storage location
             const rawResult = inst.results || {};
             
-            // Handle Multi-Date Fan-Out (Transposition)
-            // Logic: Checks if result is { userId: { date: data } }
             const firstUser = Object.keys(rawResult)[0];
             let dataToCommit = rawResult;
-            let isMultiDate = false;
 
             if (firstUser && rawResult[firstUser] && typeof rawResult[firstUser] === 'object') {
                 const innerKeys = Object.keys(rawResult[firstUser]);
                 if (innerKeys.length > 0 && innerKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k))) {
-                    isMultiDate = true;
                     const transposed = {};
                     for (const [userId, dateMap] of Object.entries(rawResult)) {
                         for (const [dateKey, dailyData] of Object.entries(dateMap)) {
@@ -180,24 +165,22 @@ class StandardExecutor {
                 }
             }
 
-            // Create a mock instance for the committer that returns just this batch
             transformedState[name] = {
                 manifest: inst.manifest,
                 getResult: async () => dataToCommit,
-                _executionStats: executionStats[name] // Attach current stats including timings
+                _executionStats: executionStats[name]
             };
 
-            // ⚠️ CRITICAL: CLEAR MEMORY
             inst.results = {};
         }
 
-        // 2. Commit Batch
+        // [UPDATED] Pass isInitialWrite to ResultCommitter
         const result = await commitResults(transformedState, dateStr, passName, config, deps, skipStatusWrite, {
-            flushMode: mode,      // 'INTERMEDIATE' or 'FINAL'
-            shardIndexes: shardIndexMap // Pass the tracking map
+            flushMode: mode,      
+            shardIndexes: shardIndexMap,
+            isInitialWrite: isInitialWrite 
         });
 
-        // 3. Update Shard Indexes from result
         if (result.shardIndexes) {
             Object.assign(shardIndexMap, result.shardIndexes);
         }
@@ -208,19 +191,16 @@ class StandardExecutor {
     static mergeReports(successAcc, failureAcc, newResult) {
         if (!newResult) return;
         
-        // Merge Success Updates (Sums metrics)
         for (const [name, update] of Object.entries(newResult.successUpdates)) {
             if (!successAcc[name]) {
                 successAcc[name] = update;
             } else {
-                // Sum storage metrics
                 if (update.metrics?.storage) {
                     successAcc[name].metrics.storage.sizeBytes += (update.metrics.storage.sizeBytes || 0);
                     successAcc[name].metrics.storage.keys      += (update.metrics.storage.keys || 0);
                     successAcc[name].metrics.storage.shardCount = Math.max(successAcc[name].metrics.storage.shardCount, update.metrics.storage.shardCount || 1);
                 }
                 
-                // [IDEA 2] Sum timing metrics
                 if (update.metrics?.execution?.timings) {
                      if (!successAcc[name].metrics.execution) successAcc[name].metrics.execution = { timings: { setup:0, stream:0, processing:0 }};
                      const tDest = successAcc[name].metrics.execution.timings;
@@ -230,13 +210,10 @@ class StandardExecutor {
                      tDest.stream += (tSrc.stream || 0);
                      tDest.processing += (tSrc.processing || 0);
                 }
-
-                // Keep the latest hash/composition info
                 successAcc[name].hash = update.hash;
             }
         }
 
-        // Merge Failures
         if (newResult.failureReport) {
             failureAcc.push(...newResult.failureReport);
         }
@@ -278,4 +255,4 @@ class StandardExecutor {
     }
 }
 
-module.exports = { StandardExecutor };
+module.exports = { StandardExecutor };