bulltrackers-module 1.0.133 → 1.0.134

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -23,69 +23,34 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
     const passToRun = String(config.COMPUTATION_PASS_TO_RUN); if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
-
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    
-    // --- MODIFIED: Call new date function ---
     const earliestDates = await getEarliestDataDates(config, dependencies);
-    const firstDate = earliestDates.absoluteEarliest; // Use the absolute earliest for the loop
-    // --- END MODIFICATION ---
-    
+    const firstDate = earliestDates.absoluteEarliest; 
     const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-
     const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
     const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
-
-    // --- NEW: Helper function to process a single date ---
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr+'T00:00:00Z');
         try {
-            // --- MODIFIED: Pass earliestDates map to checkRootDataAvailability ---
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-            if (!rootData) {
-                logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);
-                return; // Use 'return' instead of 'continue' for a mapped function
-            }
-            // --- END MODIFICATION ---
-            
-            // --- MODIFIED: Fetch ALL existing results for this pass, not just dependencies ---
+            if (!rootData) { logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);return;}
             const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            
-            // --- MODIFIED: Pass existingResults to filterCalculations ---
             const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger);
-            
-            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {
-                logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);
-                return;
-            }
-
-            // --- MODIFIED: Pass existingResults (as fetchedDeps) to meta pass ---
+            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);return;}
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
             if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
-            
-            // Note: We no longer log "Completed" here, as the sub-functions do.
-            // logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
-
-        } catch (err) {
-            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        }
+        } catch (err) {logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });}
     };
-    // --- END: Helper function ---
-
-    // --- NEW: Replace sequential loop with parallel batch loop ---
     logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates in batches of ${PARALLEL_BATCH_SIZE}...`);
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
         logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
         await Promise.all(batch.map(dateStr => processDate(dateStr)));
     }
-    // --- END: Parallel batch loop ---
-
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -55,21 +55,16 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
  */
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    
-    // --- MODIFIED: Check all calcs in this pass, not just meta-deps ---
     const allCalcsInPass = new Set(calcsInPass.map(c => normalizeName(c.name)));
     if (!allCalcsInPass.size) return {};
-
     logger.log('INFO', `[PassRunner] Checking for ${allCalcsInPass.size} existing results for ${dateStr}...`);
     const docRefs = [], depNames = [];
-    
     for (const calcName of allCalcsInPass) {
         const calcManifest = manifestMap.get(calcName);
         if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for ${calcName}`); continue; }
         docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category||'unknown').collection(config.computationsSubcollection).doc(calcName));
         depNames.push(calcName);
     }
-    
     const fetched = {};
     if (docRefs.length) (await db.getAll(...docRefs)).forEach((doc,i)=>fetched[depNames[i]]=doc.exists?doc.data():null);
     return fetched;
@@ -79,51 +74,24 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
  */
 function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger) {
     const skipped = new Set();
-
     // Filter Standard Calcs
     const standardCalcsToRun = standardCalcs.filter(c => {
-        // --- NEW: Skip if result already exists ---
-        if (existingResults[c.name]) {
-            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);
-            return false;
-        }
-        
+        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);return false;}
         const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
         if (canRun) return true;
-        
         logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
         skipped.add(c.name);
         return false;
     });
-
     // Filter Meta Calcs
     const metaCalcsToRun = metaCalcs.filter(c => {
-        // --- NEW: Skip if result already exists ---
-        if (existingResults[c.name]) {
-            logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);
-            skipped.add(c.name);
-            return false;
-        }
-
+        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);skipped.add(c.name);return false;}
         // 1. Check root data
         const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
-        if (!canRun) {
-            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);
-            skipped.add(c.name);
-            return false;
-        }
-        
+        if (!canRun) {logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);skipped.add(c.name);return false;}
         // 2. Check computed dependencies
-        // --- MODIFIED: Check existingResults (was fetchedDeps) ---
         const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
-        if (missingDeps.length > 0) {
-            logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
-            skipped.add(c.name);
-            return false;
-        }
-        return true; // All checks passed
-    });
-    
+        if (missingDeps.length > 0) {logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);skipped.add(c.name);return false;} return true;});
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
@@ -146,91 +114,53 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) { const u
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
-    
     let firstUser=true;
-    // Load mappings once for all chunks
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
-
-    // Use the new async generator stream
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
-        // This inner loop processes one chunk of users at a time
-        for(const uid in chunk){ 
-            const p=chunk[uid]; if(!p) continue; 
+        for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
             context.userType=userType; 
             for(const name in state){ 
                 const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
                 const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
-                // --- MODIFIED: Argument list now includes context ---
                 let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
-                
                 if(isSocialOrInsights&&!firstUser) continue; 
-                
-                if(isHistorical){ 
-                    const pY=yesterdayPortfolios[uid]; if(!pY) continue; 
-                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
-                } 
-                
+                if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } 
                 if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
-                
-                try{ 
-                    // Pass context to all process functions.
-                    await Promise.resolve(calc.process(...args)); 
-                } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} 
-            } 
+                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
             firstUser=false; 
         } 
-        // Memory from 'chunk' is released here before the next iteration
     }
 }
-// --- END MODIFICATION ---
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    // --- MODIFIED: Log only if there are calcs to run ---
     if (calcs.length === 0) return; 
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    
-    // --- MODIFIED: We no longer need portfolioRefs in fullRoot, as streamAndProcess handles it.
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    
-    // --- MODIFIED: Call to streamAndProcess no longer passes portfolioRefs ---
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
-    
-    // --- START: FULL COMMIT LOGIC ---
     let success = 0;
     const standardWrites = [];
-    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
-
+    const shardedWrites = {}; 
     for (const name in state) {
         const calc = state[name];
         if (!calc || typeof calc.getResult !== 'function') continue;
-        
-        try {
-            const result = await Promise.resolve(calc.getResult());
+        try {const result = await Promise.resolve(calc.getResult());
             if (result && Object.keys(result).length > 0) {
-                
-                // Separate sharded data from standard data
                 const standardResult = {};
                 for (const key in result) {
                     if (key.startsWith('sharded_')) {
-                        // This is sharded data, e.g., sharded_user_profitability
-                        // The value is expected to be: { "collection_name": { "doc1": {...}, "doc2": {...} } }
                         const shardedData = result[key];
                         for (const collectionName in shardedData) {
                             if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            // Merge doc data (e.g., combining data for "user_profitability_shard_1")
                             Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
                         }
                     } else {
-                        // This is a standard, single-doc result
                         standardResult[key] = result[key];
                     }
                 }
-
-                // Add standard result to the batch
                 if (Object.keys(standardResult).length > 0) {
                     const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
                         .collection(config.resultsSubcollection).doc(calc.manifest.category)
@@ -238,82 +168,70 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
                     
                     standardWrites.push({ ref: docRef, data: standardResult });
                 }
-                success++; // Mark as success even if only sharded data was produced
+                success++;
             }
         } catch (e) {
             logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
         }
     }
-    
-    // Commit standard (non-sharded) writes in chunks
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
     }
-
-    // Commit all sharded writes
-    for (const collectionName in shardedWrites) {
-        const docs = shardedWrites[collectionName];
+    for (const docPath in shardedWrites) { // 'docPath' is the key, e.g., 'user_profile_history_shard_0' or 'social_.../history'
+        const docData = shardedWrites[docPath]; // 'docData' is the object to write, e.g., { profiles: ... }
         const shardedDocWrites = [];
-        for (const docId in docs) {
-            // This assumes docId is the full path for sharded docs, or just the doc ID
-            // Based on user_profitability_tracker, it's just the doc ID.
-            const docRef = deps.db.collection(collectionName).doc(docId);
-            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
+        let docRef;
+        if (docPath.includes('/')) {
+            // Path is absolute, e.g., 'social_prediction_regime_state/history'
+            docRef = deps.db.doc(docPath);
+        } else {
+            // Path is a docId, e.g., 'user_profile_history_shard_0'
+            // We must infer its collection from config.
+            const collection = (docPath.startsWith('user_profile_history')) 
+                ? config.shardedUserProfileCollection // 'user_profile_history'
+                : config.shardedProfitabilityCollection; // Fallback
+            docRef = deps.db.collection(collection).doc(docPath);
+        }
+        // Ensure data is a valid object before pushing
+        if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
+             shardedDocWrites.push({ ref: docRef, data: docData });
+        } else {
+             logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData });
         }
+        // Commit this single document write (or small batch if logic is changed later)
         if (shardedDocWrites.length > 0) {
-            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
+            // Use the docPath in the operation name for clearer logging
+            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`);
         }
     }
-    // --- END: FULL COMMIT LOGIC ---
-
     logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    // --- MODIFIED: Log only if there are calcs to run ---
     if (calcs.length === 0) return;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-
-    // --- START: FULL COMMIT LOGIC ---
     let success = 0;
     const standardWrites = [];
-    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
-
+    const shardedWrites = {}; 
     for (const mCalc of calcs) {
         const name = normalizeName(mCalc.name), Cl = mCalc.class;
-        if (typeof Cl !== 'function') {
-            logger.log('ERROR', `Invalid class ${name}`);
-            continue;
-        }
+        if (typeof Cl !== 'function') {logger.log('ERROR', `Invalid class ${name}`);continue;}
         const inst = new Cl();
         try {
-            // Pass the full dependencies object to process()
             const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData }, config, fetchedDeps));
-            
-            if (result && Object.keys(result).length > 0) {
-                
-                // Separate sharded data from standard data
-                const standardResult = {};
-                for (const key in result) {
-                    if (key.startsWith('sharded_')) {
-                        const shardedData = result[key];
-                        for (const collectionName in shardedData) {
-                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
-                        }
+            if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
+                    if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 
+                        {if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};Object.assign(shardedWrites[collectionName], shardedData[collectionName]);}
                     } else {
                         standardResult[key] = result[key];
                     }
                 }
-
-                // Add standard result to the batch
                 if (Object.keys(standardResult).length > 0) {
                     const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
                         .collection(config.resultsSubcollection).doc(mCalc.category)
                         .collection(config.computationsSubcollection).doc(name);
-                    
                     standardWrites.push({ ref: docRef, data: standardResult });
                 }
                 success++;
@@ -322,33 +240,23 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
         }
     }
-
-    // Commit standard (non-sharded) writes in chunks
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);
     }
-
-    // Commit all sharded writes
     for (const collectionName in shardedWrites) {
         const docs = shardedWrites[collectionName];
         const shardedDocWrites = [];
         for (const docId in docs) {
-            // Special case for stateful meta-calcs that write to a specific path
             const docRef = docId.includes('/')
-                ? deps.db.doc(docId) // docId is a full path
-                : deps.db.collection(collectionName).doc(docId); // docId is just an ID
-            
+                ? deps.db.doc(docId) 
+                : deps.db.collection(collectionName).doc(docId);
             shardedDocWrites.push({ ref: docRef, data: docs[docId] });
         }
         if (shardedDocWrites.length > 0) {
             await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
         }
-    }
-    // --- END: FULL COMMIT LOGIC ---
-    
+    }    
     logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
-
-// --- MODIFIED: Export new function name ---
 module.exports = { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass };

package/functions/core/utils/firestore_utils.js

@@ -17,28 +17,17 @@ const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 async function getLatestNormalUserPortfolios(dependencies, config) {
     const { db, logger } = dependencies;
     const { normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName } = config;
-
     logger.log('INFO', `[Core Utils] Fetching latest portfolios from ${normalUserCollectionName}...`);
     const allPortfolios = {};
-
     const yesterday = new Date();
     yesterday.setDate(yesterday.getDate() - 1);
     const dateString = yesterday.toISOString().slice(0, 10);
-
     const blockDocs = await db.collection(normalUserCollectionName).listDocuments();
-
     for (const blockDoc of blockDocs) {
         const snapshotDocRef = blockDoc.collection(snapshotsSubCollectionName).doc(dateString);
         const partsCollectionRef = snapshotDocRef.collection(partsSubCollectionName);
         const partsSnapshot = await partsCollectionRef.get();
-
-        if (!partsSnapshot.empty) {
-            partsSnapshot.forEach(partDoc => {
-                Object.assign(allPortfolios, partDoc.data());
-            });
-        }
-    }
-
+        if (!partsSnapshot.empty) {partsSnapshot.forEach(partDoc => {Object.assign(allPortfolios, partDoc.data());});}}
     logger.log('INFO', `[Core Utils] Found ${Object.keys(allPortfolios).length} user portfolios from ${dateString}'s snapshot.`);
     return allPortfolios;
 }
@@ -53,27 +42,14 @@ async function getLatestNormalUserPortfolios(dependencies, config) {
 async function resetProxyLocks(dependencies, config) {
     const { db, logger } = dependencies;
     const { proxyPerformanceDocPath } = config;
-
     logger.log('INFO','[Core Utils] Resetting proxy locks...');
-    try {
-        if (!proxyPerformanceDocPath) {
-            logger.log('ERROR', '[Core Utils] Missing proxyPerformanceDocPath. Cannot reset locks.');
-            return;
-        }
-        
+    try {if (!proxyPerformanceDocPath) {logger.log('ERROR', '[Core Utils] Missing proxyPerformanceDocPath. Cannot reset locks.');return;}
         const perfDocRef = db.doc(proxyPerformanceDocPath);
-
-        await perfDocRef.update({
-            locks: FieldValue.delete() 
-        });
-
+        await perfDocRef.update({locks: FieldValue.delete() });
         logger.log('INFO',`[Core Utils] Proxy locks map reset in ${proxyPerformanceDocPath}.`);
     } catch (error) {
-        if (error.code === 5) { // 5 = NOT_FOUND
-             logger.log('WARN',`[Core Utils] Proxy performance doc or 'locks' field not found at ${proxyPerformanceDocPath}. No locks to reset.`);
-        } else {
-            logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message, path: proxyPerformanceDocPath });
-        }
+        if (error.code === 5) { logger.log('WARN',`[Core Utils] Proxy performance doc or 'locks' field not found at ${proxyPerformanceDocPath}. No locks to reset.`);
+        } else {logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message, path: proxyPerformanceDocPath });}
     }
 }
 
@@ -89,29 +65,15 @@ async function resetProxyLocks(dependencies, config) {
 async function getBlockCapacities(dependencies, config, userType) {
     const { db, logger } = dependencies;
     const { speculatorBlockCountsDocPath, normalBlockCountsDocPath } = config;
-
     logger.log('INFO',`[Core Utils] Getting block capacities for ${userType}...`);
     try {
-        const docPath = userType === 'speculator'
-            ? speculatorBlockCountsDocPath
-            : normalBlockCountsDocPath;
-            
-        if (!docPath) {
-             logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);
-             return {};
-        }
-
+        const docPath = userType === 'speculator'? speculatorBlockCountsDocPath : normalBlockCountsDocPath;
+        if (!docPath) {logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);return {};}
         const countsRef = db.doc(docPath);
         const countsDoc = await countsRef.get();
-        if (!countsDoc.exists) {
-            logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);
-            return {};
-        }
+        if (!countsDoc.exists) {logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);return {};}
         return countsDoc.data().counts || {};
-    } catch (error) {
-        logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });
-        throw error;
-    }
+    } catch (error) {logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });throw error;}
 }
 
 /**
@@ -127,63 +89,30 @@ async function getBlockCapacities(dependencies, config, userType) {
  */
 async function getExclusionIds(dependencies, config, userType) {
     const { db, logger } = dependencies;
-    const { 
-        specBlocksCollection, 
-        pendingSpecCollection, 
-        invalidSpecCollection,
-        existingNormalUserIds // Get the pre-fetched IDs
-    } = config;
-    
+    const { specBlocksCollection, pendingSpecCollection, invalidSpecCollection,existingNormalUserIds } = config;
     logger.log('INFO',`[Core Utils] Getting exclusion IDs for ${userType} discovery...`);
-    
     const exclusionIds = new Set(existingNormalUserIds);
     logger.log('TRACE', `[Core Utils] Loaded ${exclusionIds.size} existing normal user IDs for exclusion.`);
-    
     const promises = [];
-
     try {
         // 1. Existing Speculators
         const specBlocksRef = db.collection(specBlocksCollection);
-        promises.push(specBlocksRef.get().then(snapshot => {
-            snapshot.forEach(doc => {
-                 const users = doc.data().users || {};
-                 Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1]));
-            });
-            logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');
-        }));
-
+        promises.push(specBlocksRef.get().then(snapshot => {snapshot.forEach(doc => {const users = doc.data().users || {}; Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1])); });
+        logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');}));
         // 2. Pending Speculators
-        if (userType === 'speculator') {
-            const pendingRef = db.collection(pendingSpecCollection);
-             promises.push(pendingRef.get().then(snapshot => {
-                snapshot.forEach(doc => {
-                    Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
-                });
-                logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');
-             }));
-        }
-
+        if (userType === 'speculator') {const pendingRef = db.collection(pendingSpecCollection);
+        promises.push(pendingRef.get().then(snapshot => {snapshot.forEach(doc => {Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));});
+        logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');})); }
         // 3. Invalid Speculators
         const invalidRef = db.collection(invalidSpecCollection);
-         promises.push(invalidRef.get().then(snapshot => {
-             snapshot.forEach(doc => {
-                 const data = doc.data();
-                 if (data) {
-                    Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));
-                 }
-             });
-              logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');
-         }));
-
+        promises.push(invalidRef.get().then(snapshot => { snapshot.forEach(doc => {const data = doc.data();if (data) {Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));}});
+        logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');}));
         await Promise.all(promises);
         logger.log('INFO',`[Core Utils] Total unique exclusion IDs found: ${exclusionIds.size}`);
         return exclusionIds;
-
-    } catch (error)
- {
+    } catch (error){
         logger.log('ERROR','[Core Utils] Error getting exclusion IDs', { errorMessage: error.message });
-        throw error;
-    }
+        throw error; }
 }
 
 /**
@@ -198,20 +127,12 @@ async function getPrioritizedSpeculators(dependencies, exclusionIds, speculatorI
     const { logger } = dependencies;
     logger.log('INFO','[Core Utils] Scanning normal users for prioritized speculators...');
     const candidates = new Set();
-
     try {
         for (const userId in latestNormalPortfolios) {
             if (exclusionIds.has(userId)) continue;
-
             const portfolio = latestNormalPortfolios[userId];
-            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p =>
-                speculatorInstrumentSet.has(p.InstrumentID)
-            );
-
-            if (holdsSpeculatorAsset) {
-                candidates.add(userId);
-            }
-        }
+            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p => speculatorInstrumentSet.has(p.InstrumentID));
+            if (holdsSpeculatorAsset) { candidates.add(userId);}}
         logger.log('INFO',`[Core Utils] Found ${candidates.size} potential prioritized speculators.`);
         return Array.from(candidates);
     } catch (error) {
@@ -235,23 +156,15 @@ async function clearCollection(dependencies, collectionPath, maxBatchSize = 400)
         let query = collectionRef.limit(maxBatchSize);
         let snapshot;
         let deleteCount = 0;
-
         while (true) {
             snapshot = await query.get();
-            if (snapshot.size === 0) {
-                break; 
-            }
-
+            if (snapshot.size === 0) { break; }
             const batch = db.batch();
             snapshot.docs.forEach(doc => batch.delete(doc.ref));
             await batch.commit();
             deleteCount += snapshot.size;
-
-            if (snapshot.size < maxBatchSize) {
-                break; 
-            }
-             query = collectionRef.limit(maxBatchSize); 
-        }
+            if (snapshot.size < maxBatchSize) {break; }
+             query = collectionRef.limit(maxBatchSize); }
         logger.log('SUCCESS', `[Core Utils] Scorched earth complete. Deleted ${deleteCount} documents from ${collectionPath}.`);
     } catch (error) {
         logger.log('ERROR', `[Core Utils] Error clearing collection ${collectionPath}`, { errorMessage: error.message });
@@ -272,17 +185,9 @@ async function clearCollection(dependencies, collectionPath, maxBatchSize = 400)
  */
 async function batchWriteShardedIds(dependencies, config) {
     const { db, logger } = dependencies;
-    const {
-        collectionPath,
-        items,
-        timestamp,
-        maxFieldsPerDoc,
-        maxWritesPerBatch
-    } = config;
-
+    const {collectionPath, items, timestamp, maxFieldsPerDoc, maxWritesPerBatch } = config;
     logger.log('INFO', `[Core Utils] Batch writing ${items.length} IDs to sharded path: ${collectionPath} (max ${maxFieldsPerDoc}/doc, ${maxWritesPerBatch} docs/batch)...`);
     if (items.length === 0) return;
-
     try {
         const collectionRef = db.collection(collectionPath);
         let batch = db.batch();
@@ -290,27 +195,19 @@ async function batchWriteShardedIds(dependencies, config) {
         let currentFieldCount = 0;
         let batchWriteCount = 0;
         let docCounter = 0;
-
         for (let i = 0; i < items.length; i++) {
             const userId = items[i];
             const key = `users.${userId}`; 
             currentDocFields[key] = timestamp;
             currentFieldCount++;
-
             if (currentFieldCount >= maxFieldsPerDoc || i === items.length - 1) {
                 const docRef = collectionRef.doc(`pending_${docCounter}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`);
                 batch.set(docRef, currentDocFields); 
                 batchWriteCount++;
-
                 currentDocFields = {};
                 currentFieldCount = 0;
                 docCounter++;
-
-                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) {
-                    await batch.commit();
-                    batch = db.batch(); 
-                    batchWriteCount = 0;
-                }
+                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) { await batch.commit(); batch = db.batch(); batchWriteCount = 0; }
             }
         }
          logger.log('SUCCESS', `[Core Utils] Sharded write complete for ${collectionPath}. Created ${docCounter} documents.`);
@@ -336,24 +233,11 @@ async function getNormalUsersToUpdate(dependencies, config) {
     logger.log('INFO','[Core Utils] Getting normal users to update...');
     const usersToUpdate = [];
     try {
-        const timestampDocRef = db.collection(normalUserCollectionName)
-                                 .doc('timestamps')
-                                 .collection('users')
-                                 .doc(normalUserTimestampsDocId); 
+        const timestampDocRef = db.collection(normalUserCollectionName) .doc('timestamps') .collection('users') .doc(normalUserTimestampsDocId); 
         const timestampDoc = await timestampDocRef.get();
-
-        if (!timestampDoc.exists) {
-            logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`);
-            return [];
-        }
-
+        if (!timestampDoc.exists) { logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`); return []; }
         const timestamps = timestampDoc.data().users || {};
-        for (const userId in timestamps) {
-            const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0);
-            if (lastProcessed < dateThreshold) {
-                usersToUpdate.push(userId);
-            }
-        }
+        for (const userId in timestamps) { const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0); if (lastProcessed < dateThreshold) { usersToUpdate.push(userId); }}
         logger.log('INFO',`[Core Utils] Found ${usersToUpdate.length} normal users to update.`);
         return usersToUpdate;
     } catch (error) {
@@ -374,44 +258,26 @@ async function getNormalUsersToUpdate(dependencies, config) {
 async function getSpeculatorsToUpdate(dependencies, config) {
     const { db, logger } = dependencies;
     const { dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName } = config;
-
     logger.log('INFO','[Core Utils] Getting speculators to update...');
     const updates = [];
     try {
         const blocksRef = db.collection(speculatorBlocksCollectionName);
         const snapshot = await blocksRef.get();
-
-        if (snapshot.empty) {
-            logger.log('INFO','[Core Utils] No speculator blocks found.');
-            return [];
-        }
-
+        if (snapshot.empty) { logger.log('INFO','[Core Utils] No speculator blocks found.'); return []; }
         snapshot.forEach(doc => {
             const blockData = doc.data();
-
-            // Iterate over the document's top-level keys
             for (const key in blockData) {
-                 // Filter for keys that match the 'users.CID' format
                  if (!key.startsWith('users.')) continue; 
-
                  const userId = key.split('.')[1];
-                 if (!userId) continue; // Safety check
-
-                 const userData = blockData[key]; // Get the user's map
-
+                 if (!userId) continue; 
+                 const userData = blockData[key]; 
                  const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
                  const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
-
-                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
-                     if (userData.instruments && Array.isArray(userData.instruments)) {
-                         userData.instruments.forEach(instrumentId => {
-                             updates.push({ userId, instrumentId });
-                         });
+                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) { if (userData.instruments && Array.isArray(userData.instruments)) { userData.instruments.forEach(instrumentId => { updates.push({ userId, instrumentId }); });
                      }
                  }
             }
         });
-
         logger.log('INFO',`[Core Utils] Found ${updates.length} speculator user/instrument pairs to update.`);
         return updates;
     } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.133",
+  "version": "1.0.134",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [