bulltrackers-module 1.0.181 → 1.0.183

package/functions/computation-system/controllers/computation_controller.js

@@ -1,6 +1,6 @@
 /**
  * FIXED: computation_controller.js
- * V3.2: Supports Streaming Trading History (todayHistory) separate from Yesterday's Portfolio.
+ * V3.3: Adds Price Loading for Meta Context & Fixes Context Injection
  */
 
 const { DataExtractor, 
@@ -23,7 +23,7 @@ class DataLoader {
     constructor(config, dependencies) {
         this.config = config;
         this.deps = dependencies;
-        this.cache = { mappings: null, insights: new Map(), social: new Map() };
+        this.cache = { mappings: null, insights: new Map(), social: new Map(), prices: null };
     }
     async loadMappings() {
         if (this.cache.mappings) return this.cache.mappings;
@@ -43,6 +43,48 @@ class DataLoader {
         this.cache.social.set(dateStr, social);
         return social;
     }
+    /**
+     * NEW: Loads sharded price data for Meta calculations
+     */
+    async loadPrices() {
+        if (this.cache.prices) return this.cache.prices;
+        const { db, logger } = this.deps;
+        const collection = this.config.priceCollection || 'asset_prices';
+        
+        logger.log('INFO', `[DataLoader] Loading all price shards from ${collection}...`);
+        
+        try {
+            const snapshot = await db.collection(collection).get();
+            if (snapshot.empty) return { history: [] };
+
+            // Flatten shards into a single array for the context
+            // Structure expected by calculation: Array of { instrumentId, prices: {...} }
+            const allPrices = [];
+            
+            snapshot.forEach(doc => {
+                const shardData = doc.data();
+                // Iterate keys in shard (instrumentIds)
+                for (const [instId, data] of Object.entries(shardData)) {
+                    if (data && data.prices) {
+                        allPrices.push({
+                            instrumentId: instId,
+                            ...data 
+                        });
+                    }
+                }
+            });
+
+            logger.log('INFO', `[DataLoader] Loaded prices for ${allPrices.length} instruments.`);
+            
+            // Cache as an object with 'history' array to match context expectations
+            this.cache.prices = { history: allPrices };
+            return this.cache.prices;
+
+        } catch (e) {
+            logger.log('ERROR', `[DataLoader] Failed to load prices: ${e.message}`);
+            return { history: [] };
+        }
+    }
 }
 
 class ContextBuilder {
@@ -76,7 +118,7 @@ class ContextBuilder {
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
             mappings: mappings || {},
-            math: { // Introduced a new computation class in the math primitives? Add it here. Then add it to meta context a little further down.
+            math: { 
                 extract: DataExtractor,
                 history: HistoryExtractor,
                 compute: MathPrimitives,
@@ -103,6 +145,7 @@ class ContextBuilder {
             mappings, 
             insights, 
             socialData,
+            prices, // <--- ADDED THIS
             computedDependencies, 
             previousComputedDependencies,
             config, 
@@ -113,8 +156,9 @@ class ContextBuilder {
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
             social: { today: socialData?.today, yesterday: socialData?.yesterday },
+            prices: prices || {}, // <--- INJECTED HERE
             mappings: mappings || {},
-            math: { // Introduced a new computation class in the math primitives? Add it here.
+            math: { 
                 extract: DataExtractor,
                 history: HistoryExtractor,
                 compute: MathPrimitives,
@@ -142,22 +186,22 @@ class ComputationExecutor {
         this.loader = dataLoader;
     }
 
-    /**
-     * UPDATED: Accepts yesterdayPortfolioData AND historyData separately.
-     */
     async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps) {
         const { logger } = this.deps;
         const targetUserType = metadata.userType; 
         const mappings = await this.loader.loadMappings();
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
+        
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
             const todayHistory = historyData ? historyData[userId] : null;
             const actualUserType = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
+            
             if (targetUserType !== 'all') {
                  const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
                  if (mappedTarget !== actualUserType) continue;
             }
+            
             const context = ContextBuilder.buildPerUserContext({
                 todayPortfolio, yesterdayPortfolio,
                 todayHistory, 
@@ -166,6 +210,7 @@ class ComputationExecutor {
                 previousComputedDependencies: prevDeps,
                 config: this.config, deps: this.deps
             });
+            
             try { await calcInstance.process(context); } 
             catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
         }
@@ -173,10 +218,20 @@ class ComputationExecutor {
 
     async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps) {
         const mappings = await this.loader.loadMappings();
+        
+        // Load standard dependencies
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
         const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
+        
+        // NEW: Load Price dependencies if required
+        let prices = null;
+        if (metadata.rootDataDependencies?.includes('price')) {
+            prices = await this.loader.loadPrices();
+        }
+
         const context = ContextBuilder.buildMetaContext({
             dateStr, metadata, mappings, insights, socialData: social,
+            prices, // Pass prices to builder
             computedDependencies: computedDeps,
             previousComputedDependencies: prevDeps,
             config: this.config, deps: this.deps

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -6,13 +6,15 @@ const {
     groupByPass, 
     checkRootDataAvailability, 
     fetchExistingResults, 
-    fetchGlobalComputationStatus,  // <--- New Import
-    updateGlobalComputationStatus, // <--- New Import
+    fetchComputationStatus,   
+    updateComputationStatus,
     runStandardComputationPass, 
     runMetaComputationPass,
     checkRootDependencies
 } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings } = require('../utils/utils.js');
+
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+
 const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
@@ -21,7 +23,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!passToRun) 
         return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Global Status Check...`);
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
 
     // Hardcoded earliest dates
     const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
@@ -34,42 +36,53 @@ async function runComputationPass(config, dependencies, computationManifest) {
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
     
     const passEarliestDate = earliestDates.absoluteEarliest; 
-    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
     const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
     const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
 
-    // 1. Fetch Global Status ONCE (Memory Cache)
-    // Returns { "2023-10-27": { calcA: true, calcB: false }, ... }
-    const globalStatusData = await fetchGlobalComputationStatus(config, dependencies);
-
-    // Helper: Check status using in-memory data
-    const shouldRun = (calc, dateStr) => {
-        const dailyStatus = globalStatusData[dateStr] || {};
-        
-        // 1. If explicitly TRUE, ignore.
-        if (dailyStatus[calc.name] === true) return false;
-
-        // 2. Check dependencies (using same in-memory status)
-        if (calc.dependencies && calc.dependencies.length > 0) {
-            const depsMet = calc.dependencies.every(depName => dailyStatus[depName] === true);
-            if (!depsMet) return false; 
-        }
-        return true;
-    };
-
-    // Process a single date and RETURN updates (do not write)
+    // Process a single date
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-        const standardToRun = standardCalcs.filter(c => shouldRun(c, dateStr));
-        const metaToRun     = metaCalcs.filter(c => shouldRun(c, dateStr));
+
+        // 1. Fetch Status for THIS specific date only
+        // This ensures Pass 2 sees exactly what Pass 1 wrote for this date.
+        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+        // Helper: Check status using the fetched daily data
+        const shouldRun = (calc) => {
+            const cName = normalizeName(calc.name);
+            
+            // A. If recorded as TRUE -> Ignore (already ran)
+            if (dailyStatus[cName] === true) return false;
+
+            // B. If recorded as FALSE or UNDEFINED -> Run it (retry or new)
+            // But first, check if we have the necessary data dependencies.
+            
+            if (calc.dependencies && calc.dependencies.length > 0) {
+                // Check if prerequisites (from previous passes on THIS date) are complete
+                const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
+                if (missing.length > 0) {
+                    // Dependency missing: cannot run yet.
+                    return false;
+                }
+            }
+            
+            // If we are here, status is false/undefined AND dependencies are met.
+            return true;
+        };
+
+        const standardToRun = standardCalcs.filter(shouldRun);
+        const metaToRun     = metaCalcs.filter(shouldRun);
         
-        if (!standardToRun.length && !metaToRun.length) return null; // No work
+        if (!standardToRun.length && !metaToRun.length) return null; // No work for this date
 
+        // 2. Check Root Data Availability (Portfolio, History, etc.)
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
         if (!rootData) return null;
 
+        // 3. Filter again based on Root Data availability
         const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
         const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
@@ -85,52 +98,35 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
+            
+            // Note: We use skipStatusWrite=true because we want to batch write the status at the end of this function
             if (finalStandardToRun.length) {
-                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); // skipStatusWrite=true
+                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); 
                 Object.assign(dateUpdates, updates);
             }
             if (finalMetaToRun.length) {
-                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); // skipStatusWrite=true
+                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); 
                 Object.assign(dateUpdates, updates);
             }
         } catch (err) { 
             logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-            // Mark failures
-            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[c.name] = false);
+            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
+        }
+
+        // 4. Write "true" or "false" results for THIS specific date immediately
+        if (Object.keys(dateUpdates).length > 0) {
+            await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
         }
 
-        // Return the updates for this date
         return { date: dateStr, updates: dateUpdates };
     };
 
     // Batch process dates
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        
-        // Run batch in parallel
-        const results = await Promise.all(batch.map(processDate));
-        
-        // Aggregate updates from the batch
-        const batchUpdates = {};
-        let hasUpdates = false;
-        
-        results.forEach(res => {
-            if (res && res.updates && Object.keys(res.updates).length > 0) {
-                batchUpdates[res.date] = res.updates;
-                hasUpdates = true;
-                
-                // Also update our local in-memory copy so subsequent logic in this run sees it (though passes usually rely on prev days)
-                if (!globalStatusData[res.date]) globalStatusData[res.date] = {};
-                Object.assign(globalStatusData[res.date], res.updates);
-            }
-        });
-
-        // Write status ONCE per batch
-        if (hasUpdates) {
-            await updateGlobalComputationStatus(batchUpdates, config, dependencies);
-            logger.log('INFO', `[PassRunner] Batched status update for ${Object.keys(batchUpdates).length} dates.`);
-        }
+        await Promise.all(batch.map(processDate));
     }
+    
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,5 +1,6 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
+ * FIXED: Only marks computations as TRUE if they actually store results.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -27,6 +28,8 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
         else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
         else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
+        // Note: 'price' is typically not a blocking root check in this specific function logic unless added,
+        // but usually prices are treated as auxiliary. If you want to block on prices, add it here.
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -35,7 +38,6 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
  * Checks for the availability of all required root data for a specific date.
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    // ... [Unchanged content of checkRootDataAvailability] ...
     const { logger }    = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     let portfolioRefs   = [],    historyRefs = [];
@@ -50,6 +52,7 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         
         await Promise.all(tasks);
         
+        // We allow running if ANY data is present. Specific calcs filter themselves using checkRootDependencies.
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) return null;
         
         return { 
@@ -64,10 +67,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
-/**
- * --- DEPRECATED: Old per-date fetch ---
- * Keeps compatibility but logic moves to fetchGlobalComputationStatus
- */
 async function fetchComputationStatus(dateStr, config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef     = db.collection(collection).doc(dateStr);
@@ -75,10 +74,6 @@ async function fetchComputationStatus(dateStr, config, { db }) {
     return snap.exists ? snap.data() : {};
 }
 
-/**
- * --- NEW: Fetches the SINGLE GLOBAL status document ---
- * Loads the entire history of statuses in one read.
- */
 async function fetchGlobalComputationStatus(config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef = db.collection(collection).doc('global_status');
@@ -86,9 +81,6 @@ async function fetchGlobalComputationStatus(config, { db }) {
     return snap.exists ? snap.data() : {};
 }
 
-/**
- * --- DEPRECATED: Old per-date update ---
- */
 async function updateComputationStatus(dateStr, updates, config, { db }) {
     if (!updates || Object.keys(updates).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
@@ -96,17 +88,11 @@ async function updateComputationStatus(dateStr, updates, config, { db }) {
     await docRef.set(updates, { merge: true }); 
 }
 
-/**
- * --- NEW: Batch Updates to Global Document ---
- * Accepts a map of { "YYYY-MM-DD": { calcName: true, ... } }
- * and writes them using dot notation to avoid overwriting other dates.
- */
 async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
     const docRef = db.collection(collection).doc('global_status');
     
-    // Flatten to dot notation for Firestore update: "2023-10-27.calcName": true
     const flattenUpdates = {};
     for (const [date, statuses] of Object.entries(updatesByDate)) {
         for (const [calc, status] of Object.entries(statuses)) {
@@ -117,7 +103,6 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     try {
         await docRef.update(flattenUpdates);
     } catch (err) {
-        // If doc doesn't exist (first run), update fails. Use set({merge:true}).
         if (err.code === 5) { // NOT_FOUND
              const deepObj = {};
              for (const [date, statuses] of Object.entries(updatesByDate)) {
@@ -130,10 +115,6 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     }
 }
 
-/**
- * --- UPDATED: fetchExistingResults ---
- * (Unchanged, keeps fetching results per date as this is heavy data)
- */
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
@@ -158,10 +139,6 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     return fetched;
 }
 
-/**
- * --- UPDATED: streamAndProcess ---
- * (Unchanged)
- */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
     const controller = new ComputationController(config, deps);
@@ -211,10 +188,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     logger.log('INFO', `[${passName}] Streaming complete.`);
 }
 
-/**
- * --- UPDATED: runStandardComputationPass ---
- * Now accepts `skipStatusWrite` and returns `successUpdates`
- */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
     const dStr = date.toISOString().slice(0, 10);
     const logger = deps.logger;
@@ -240,14 +213,9 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
     
-    // Return the updates instead of just writing them
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-/**
- * --- UPDATED: runMetaComputationPass ---
- * Now accepts `skipStatusWrite` and returns `successUpdates`
- */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
     const controller = new ComputationController(config, deps);
     const dStr       = date.toISOString().slice(0, 10);
@@ -266,8 +234,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 }
 
 /**
- * --- UPDATED: commitResults ---
- * Added `skipStatusWrite` parameter. Returns `successUpdates`.
+ * --- FIXED: commitResults ---
+ * Only marks 'successUpdates' if data is actually written.
  */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const writes = [], schemas = [], sharded = {};
@@ -278,13 +246,23 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         try {
             const result = await calc.getResult(); 
             if (!result) continue;
+            
             const standardRes = {};
+            let hasData = false; // Track if this calc produced data
+
             for (const key in result) {
                 if (key.startsWith('sharded_')) {
                     const sData = result[key];
-                    for (const c in sData) { sharded[c] = sharded[c] || {}; Object.assign(sharded[c], sData[c]); }
-                } else standardRes[key] = result[key];
+                    for (const c in sData) { 
+                        sharded[c] = sharded[c] || {}; 
+                        Object.assign(sharded[c], sData[c]); 
+                    }
+                    if (Object.keys(sData).length > 0) hasData = true;
+                } else {
+                    standardRes[key] = result[key];
+                }
             }
+
             if (Object.keys(standardRes).length) {
                 standardRes._completed = true;
                 writes.push({
@@ -293,7 +271,9 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                         .collection(config.computationsSubcollection).doc(name),
                     data: standardRes
                 });
+                hasData = true;
             }
+            
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
                 schemas.push({ 
@@ -304,7 +284,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 });
             }
             
-            successUpdates[name] = true;
+            // FIX: Only mark as successful if we actually had data to write.
+            if (hasData) {
+                successUpdates[name] = true;
+            } else {
+                // Optional: Log that it produced no data
+                // deps.logger.log('INFO', `Calc ${name} produced no data. Skipping status update.`);
+            }
 
         } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
     }

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -10,30 +10,31 @@ const { FieldValue } = require('@google-cloud/firestore');
 
 class FirestoreBatchManager { 
     constructor(db, headerManager, logger, config) { 
-        this.db = db; 
+        this.db            = db; 
         this.headerManager = headerManager; 
-        this.logger = logger; 
-        this.config = config; 
-        this.portfolioBatch = {}; 
-        this.timestampBatch = {}; 
-        this.tradingHistoryBatch = {}; 
+        this.logger        = logger; 
+        this.config        = config; 
+        this.portfolioBatch              = {}; 
+        this.timestampBatch              = {}; 
+        this.tradingHistoryBatch         = {}; 
         this.speculatorTimestampFixBatch = {};
 
         // Username map cache
-        this.usernameMap = new Map();
-        this.usernameMapUpdates = {};
+        this.usernameMap           = new Map();
+        this.usernameMapUpdates    = {};
         this.usernameMapLastLoaded = 0;
         
         // History fetch cache (NEW)
         this.historyFetchedUserIds = new Set();
         this.historyCacheTimestamp = Date.now();
-        this.HISTORY_CACHE_TTL_MS = config.HISTORY_CACHE_TTL_MS || 600000; 
+        this.HISTORY_CACHE_TTL_MS  = config.HISTORY_CACHE_TTL_MS || 600000; 
 
-        this.processedSpeculatorCids = new Set();
-        this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
-        this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
+        this.processedSpeculatorCids         = new Set();
+        this.usernameMapCollectionName       = config.FIRESTORE_COLLECTION_USERNAME_MAP;
+        this.normalHistoryCollectionName     = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
         this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
-        this.batchTimeout = null;
+        this.batchTimeout                    = null;
+
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
@@ -50,16 +51,10 @@ class FirestoreBatchManager {
 
     _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
 
-    // --- CRITICAL FIX: Removed aggressive timeout flush ---
-    // With sequential processing, the timer was firing too often, causing 1 write per user (expensive).
-    // Now we only flush if we hit the memory limit (MAX_BATCH_SIZE) or when explicitly called at the end.
     _scheduleFlush() {
         const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 400;
         const totalOps = this._estimateBatchSize();
-        if (totalOps >= maxBatch) { 
-            this.flushBatches(); 
-            return;
-        }
+        if (totalOps >= maxBatch) {  this.flushBatches();  return; }
     }
 
     _estimateBatchSize() {
@@ -144,54 +139,33 @@ class FirestoreBatchManager {
      */
     _flushDataBatch(batchData, firestoreBatch, logName) {
         let count = 0;
-        
-        // 1. Determine Shard Strategy
-        // If we expect ~1500 users in a block and want 200 users per shard:
-        // We need ceil(1500 / 200) = 8 shards total (part_0 to part_7).
-        // Any ID, no matter how random, will map to one of these 8 buckets.
         const TARGET_USERS = this.config.DISCOVERY_ORCHESTRATOR_TARGET_USERS_PER_BLOCK ? Number(this.config.DISCOVERY_ORCHESTRATOR_TARGET_USERS_PER_BLOCK) : 1500;
         const SHARD_CAPACITY = this.config.TASK_ENGINE_MAX_USERS_PER_SHARD ? Number(this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) : 200;
-        
-        // Ensure at least 1 shard exists
         const TOTAL_SHARDS = Math.max(1, Math.ceil(TARGET_USERS / SHARD_CAPACITY));
-
         for (const basePath in batchData) {
             const users = batchData[basePath];
             const userIds = Object.keys(users);
             if (!userIds.length) continue;
-
             const updatesByShard = {};
-
             for (const userId of userIds) {
                 const cid = parseInt(userId, 10);
                 let shardId;
 
                 if (!isNaN(cid)) {
-                    // --- MODULO SHARDING ---
-                    // Even if IDs are 10, 1000000, 500... they will round-robin into 
-                    // the fixed set of shards (e.g. 8 shards), ensuring density.
                     const shardIndex = cid % TOTAL_SHARDS;
                     shardId = `part_${shardIndex}`;
-                } else {
-                    shardId = 'part_misc';
-                }
+                } else { shardId = 'part_misc'; }
 
-                if (!updatesByShard[shardId]) {
-                    updatesByShard[shardId] = {};
-                }
+                if (!updatesByShard[shardId]) { updatesByShard[shardId] = {}; }
                 updatesByShard[shardId][userId] = users[userId];
             }
-
             for (const shardId in updatesByShard) {
                 const chunkData = updatesByShard[shardId];
                 const docRef = this.db.collection(`${basePath}/parts`).doc(shardId);
-                // merge: true ensures we append to the doc if it was started in a previous batch
                 firestoreBatch.set(docRef, chunkData, { merge: true });
                 count++;
             }
-
             this.logger.log('INFO', `[BATCH] Staged ${userIds.length} ${logName} users into ${Object.keys(updatesByShard).length} buckets (Modulo ${TOTAL_SHARDS}) for ${basePath}.`);
-            
             delete batchData[basePath];
         }
         return count;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.181",
+  "version": "1.0.183",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [