bulltrackers-module 1.0.136 → 1.0.138

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -26,11 +26,52 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
     const earliestDates = await getEarliestDataDates(config, dependencies);
-    const firstDate = earliestDates.absoluteEarliest; 
-    const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
-    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
+
+    // --- REMOVE OLD/REPLACE WITH NEW LOGIC FOR PROBLEM 3 & 4 ---
+    // const firstDate = earliestDates.absoluteEarliest; // <-- REMOVE
+    // const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z'); // <-- REMOVE
+
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+    
+    // --- START: NEW LOGIC FOR PROBLEM 3 & 4 ---
+    // Determine the earliest start date based *only* on data required for THIS pass.
+    const requiredRootData = new Set();
+    calcsInThisPass.forEach(c => {
+        (c.rootDataDependencies || []).forEach(dep => requiredRootData.add(dep));
+    });
+
+    let earliestStartDateForPass = null;
+    if (requiredRootData.size > 0) {
+        // Find the LATEST of the earliest dates for all required data types.
+        // e.g., If portfolio starts 09-25 and social starts 10-30, a calc
+        // needing both must start on 10-30.
+        let latestOfEarliestDates = new Date(0); // Start at epoch
+        requiredRootData.forEach(dep => {
+            const earliestDateForDep = earliestDates[dep]; // e.g., earliestDates.portfolio
+            if (earliestDateForDep && earliestDateForDep > latestOfEarliestDates) {
+                latestOfEarliestDates = earliestDateForDep;
+            }
+        });
+        
+        if (latestOfEarliestDates.getTime() > 0) {
+             earliestStartDateForPass = latestOfEarliestDates;
+        }
+    }
+
+    // Use the pass-specific date. Fall back to absolute earliest, then config.
+    const firstDate = earliestStartDateForPass || earliestDates.absoluteEarliest;
+    const startDateUTC = firstDate 
+        ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) 
+        : new Date(config.earliestComputationDate+'T00:00:00Z');
+    
+    logger.log('INFO', `[PassRunner] Pass ${passToRun} requires data: [${Array.from(requiredRootData).join(', ')}].`);
+    logger.log('INFO', `[PassRunner] Determined start date for this pass: ${startDateUTC.toISOString().slice(0, 10)}`);
+    // --- END: NEW LOGIC ---
+
+    const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
+    const firstDayOfBackfill = allExpectedDates.length > 0 ? allExpectedDates[0] : null; // --- MOVED FROM ABOVE ---
+    
     const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
     const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
     const processDate = async (dateStr) => {
@@ -39,7 +80,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
             if (!rootData) { logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);return;}
             const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger);
+            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger,dateStr === firstDayOfBackfill );            
             if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);return;}
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
             if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -72,20 +72,26 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
 
 /** --- MODIFIED: Stage 5: Filter calculations to skip completed work ---
  */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger) {
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger, isFirstDayOfBackfill = false) { 
     const skipped = new Set();
-    // Filter Standard Calcs
     const standardCalcsToRun = standardCalcs.filter(c => {
         if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);return false;}
-        const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
-        if (canRun) return true;
-        logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
-        skipped.add(c.name);
-        return false;
+        if (isFirstDayOfBackfill && c.isHistorical) {
+            logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
+            skipped.add(c.name);
+            return false;
+        }
     });
     // Filter Meta Calcs
     const metaCalcsToRun = metaCalcs.filter(c => {
         if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);skipped.add(c.name);return false;}
+        // --- START: RECOMMENDED ADDITION ---
+        if (isFirstDayOfBackfill && c.isHistorical) {
+            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Historical calc on the first day of backfill (no yesterday).`);
+            skipped.add(c.name);
+            return false;
+        }
+        // --- END: RECOMMENDED ADDITION ---
         // 1. Check root data
         const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
         if (!canRun) {logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);skipped.add(c.name);return false;}

package/functions/task-engine/helpers/verify_helpers.js

@@ -59,14 +59,30 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
         }
     }
 
+    // --- START FIX: MODIFIED FOR SHARDING ---
     if (Object.keys(usernameMap).length) {
-        const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc('cid_map_shard_1');
-        batch.set(mapRef, usernameMap, { merge: true });
-        logger.log('INFO', `[VERIFY] Staging ${Object.keys(usernameMap).length} username updates.`);
+        // Group updates by shard
+        const shardedUpdates = {};
+        for (const cid in usernameMap) {
+            // Re-implement the sharding logic here
+            const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
+            if (!shardedUpdates[shardId]) {
+                shardedUpdates[shardId] = {};
+            }
+            shardedUpdates[shardId][cid] = usernameMap[cid];
+        }
+
+        // Write each shard to its own document
+        for (const shardId in shardedUpdates) {
+            const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId);
+            batch.set(mapRef, shardedUpdates[shardId], { merge: true });
+        }
+        logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`);
     }
+    // --- END FIX ---
 
     await batch.commit();
     if (validUserCount) logger.log('INFO', `[VERIFY] Verified and stored ${validUserCount} new ${userType} users.`);
 }
 
-module.exports = { handleVerify };
+module.exports = { handleVerify };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -31,6 +31,13 @@ class FirestoreBatchManager {
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
+    // --- START FIX: ADDED SHARDING HELPER ---
+    _getUsernameShardId(cid) {
+        // Shard across 10 documents (supports ~200k users assuming 50 bytes/user)
+        return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
+    }
+    // --- END FIX ---
+
     _scheduleFlush() {
         if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS);
     }
@@ -40,6 +47,7 @@ class FirestoreBatchManager {
         this.usernameMap.clear();
         this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
         try {
+            // This correctly gets ALL documents (shards) in the collection
             const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
             snapshot.forEach(doc => {
                 const data = doc.data();
@@ -54,14 +62,23 @@ class FirestoreBatchManager {
 
     getUsername(cid) { return this.usernameMap.get(String(cid)); }
 
+    // --- START FIX: MODIFIED FOR SHARDING ---
     addUsernameMapUpdate(cid, username) {
         if (!username) return;
         const cidStr = String(cid);
         this.usernameMap.set(cidStr, username);
-        this.usernameMapUpdates[cidStr] = { username };
-        this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr}.`);
+
+        // Organize updates by shard ID
+        const shardId = this._getUsernameShardId(cidStr);
+        if (!this.usernameMapUpdates[shardId]) {
+            this.usernameMapUpdates[shardId] = {};
+        }
+        this.usernameMapUpdates[shardId][cidStr] = { username };
+
+        this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr} in ${shardId}.`);
         this._scheduleFlush();
     }
+    // --- END FIX ---
 
     async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
         const collection = userType === 'speculator' ? this.speculatorHistoryCollectionName : this.normalHistoryCollectionName;
@@ -151,12 +168,22 @@ class FirestoreBatchManager {
             delete this.speculatorTimestampFixBatch[docPath];
         }
 
-        if (Object.keys(this.usernameMapUpdates).length) {
-            firestoreBatch.set(this.db.collection(this.usernameMapCollectionName).doc('cid_map_shard_1'), this.usernameMapUpdates, { merge: true });
-            batchOps++;
-            this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(this.usernameMapUpdates).length} username map updates.`);
-            this.usernameMapUpdates = {};
+        // --- START FIX: MODIFIED FOR SHARDING ---
+        // Loop over each shardId in the updates object
+        for (const shardId in this.usernameMapUpdates) {
+            const updates = this.usernameMapUpdates[shardId];
+            if (updates && Object.keys(updates).length > 0) {
+                firestoreBatch.set(
+                    this.db.collection(this.usernameMapCollectionName).doc(shardId),
+                    updates,
+                    { merge: true }
+                );
+                batchOps++;
+                this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(updates).length} username updates to ${shardId}.`);
+            }
         }
+        this.usernameMapUpdates = {}; // Clear updates after staging them
+        // --- END FIX ---
 
         if (this.processedSpeculatorCids.size) {
             const cids = Array.from(this.processedSpeculatorCids);
@@ -180,4 +207,4 @@ class FirestoreBatchManager {
     }
 }
 
-module.exports = { FirestoreBatchManager };
+module.exports = { FirestoreBatchManager };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.136",
+  "version": "1.0.138",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [