bulltrackers-module 1.0.129 → 1.0.131

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -12,10 +12,17 @@ function checkRootDependencies(calcManifest, rootDataStatus) { if (!calcManifest
 }
 
 /** Stage 3: Check root data availability for a date */
-async function checkRootDataAvailability(dateStr, config, { logger, ...deps }) {
+// --- FIX: Passes the full 'dependencies' object down ---
+async function checkRootDataAvailability(dateStr, config, dependencies) {
+    const { logger } = dependencies; // Destructure logger for local use
     logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
     try {
-        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([ getPortfolioPartRefs(config, deps, dateStr), loadDailyInsights(config, deps, dateStr), loadDailySocialPostInsights(config, deps, dateStr), getHistoryPartRefs(config, deps, dateStr) ]);
+        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([
+            getPortfolioPartRefs(config, dependencies, dateStr),        // Pass full 'dependencies'
+            loadDailyInsights(config, dependencies, dateStr),         // Pass full 'dependencies'
+            loadDailySocialPostInsights(config, dependencies, dateStr), // Pass full 'dependencies'
+            getHistoryPartRefs(config, dependencies, dateStr)           // Pass full 'dependencies'
+        ]);
         const hasPortfolio = !!(portfolioRefs?.length), hasInsights = !!insightsData, hasSocial = !!socialData, hasHistory = !!(historyRefs?.length);
         if (!(hasPortfolio||hasInsights||hasSocial||hasHistory)) { logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null; }
         return { portfolioRefs: portfolioRefs||[], insightsData: insightsData||null, socialData: socialData||null, historyRefs: historyRefs||[], status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
@@ -69,20 +76,159 @@ async function streamAndProcess(dateStr, todayRefs, state, passName, config, dep
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
-    const dStr=date.toISOString().slice(0,10), logger=deps.logger;
+    const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    const fullRoot=await loadHistoricalData(date, calcs, config, deps, rootData);
-    const state=initializeCalculators(calcs, logger);
+    const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
+    const state = initializeCalculators(calcs, logger);
     await streamAndProcess(dStr, fullRoot.portfolioRefs, state, passName, config, deps, fullRoot);
-    let success=0; for(const name in state){ const calc=state[name]; if(!calc||typeof calc.getResult!=='function') continue; try{ const result=await Promise.resolve(calc.getResult()); if(result&&Object.keys(result).length){ /* Commit logic omitted for brevity */ success++; } } catch(e){logger.log('ERROR',`getResult failed ${name} for ${dStr}`,{err:e.message,stack:e.stack});} }
-    logger.log(success===calcs.length?'SUCCESS':'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
+    
+    // --- START: FULL COMMIT LOGIC ---
+    let success = 0;
+    const standardWrites = [];
+    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
+
+    for (const name in state) {
+        const calc = state[name];
+        if (!calc || typeof calc.getResult !== 'function') continue;
+        
+        try {
+            const result = await Promise.resolve(calc.getResult());
+            if (result && Object.keys(result).length > 0) {
+                
+                // Separate sharded data from standard data
+                const standardResult = {};
+                for (const key in result) {
+                    if (key.startsWith('sharded_')) {
+                        // This is sharded data, e.g., sharded_user_profitability
+                        // The value is expected to be: { "collection_name": { "doc1": {...}, "doc2": {...} } }
+                        const shardedData = result[key];
+                        for (const collectionName in shardedData) {
+                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
+                            // Merge doc data (e.g., combining data for "user_profitability_shard_1")
+                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
+                        }
+                    } else {
+                        // This is a standard, single-doc result
+                        standardResult[key] = result[key];
+                    }
+                }
+
+                // Add standard result to the batch
+                if (Object.keys(standardResult).length > 0) {
+                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                        .collection(config.computationsSubcollection).doc(name);
+                    
+                    standardWrites.push({ ref: docRef, data: standardResult });
+                }
+                success++; // Mark as success even if only sharded data was produced
+            }
+        } catch (e) {
+            logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
+        }
+    }
+    
+    // Commit standard (non-sharded) writes in chunks
+    if (standardWrites.length > 0) {
+        await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
+    }
+
+    // Commit all sharded writes
+    for (const collectionName in shardedWrites) {
+        const docs = shardedWrites[collectionName];
+        const shardedDocWrites = [];
+        for (const docId in docs) {
+            // This assumes docId is the full path for sharded docs, or just the doc ID
+            // Based on user_profitability_tracker, it's just the doc ID.
+            const docRef = deps.db.collection(collectionName).doc(docId);
+            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
+        }
+        if (shardedDocWrites.length > 0) {
+            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
+        }
+    }
+    // --- END: FULL COMMIT LOGIC ---
+
+    logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
 /** Stage 10: Run meta computations */
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) { const dStr=date.toISOString().slice(0,10), logger=deps.logger;
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
+    const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    let success=0; for(const mCalc of calcs){ const name=normalizeName(mCalc.name), Cl=mCalc.class; if(typeof Cl!=='function'){ logger.log('ERROR',`Invalid class ${name}`); continue; } const inst=new Cl(); try{ const result=await Promise.resolve(inst.process(dStr,{...deps,rootData},config,fetchedDeps)); if(result&&Object.keys(result).length){ /* Commit logic omitted */ success++; } } catch(e){logger.log('ERROR',`Meta-calc failed ${name} for ${dStr}`,{err:e.message,stack:e.stack}); } }
-    logger.log(success===calcs.length?'SUCCESS':'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
+
+    // --- START: FULL COMMIT LOGIC ---
+    let success = 0;
+    const standardWrites = [];
+    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
+
+    for (const mCalc of calcs) {
+        const name = normalizeName(mCalc.name), Cl = mCalc.class;
+        if (typeof Cl !== 'function') {
+            logger.log('ERROR', `Invalid class ${name}`);
+            continue;
+        }
+        const inst = new Cl();
+        try {
+            // Pass the full dependencies object to process()
+            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData }, config, fetchedDeps));
+            
+            if (result && Object.keys(result).length > 0) {
+                
+                // Separate sharded data from standard data
+                const standardResult = {};
+                for (const key in result) {
+                    if (key.startsWith('sharded_')) {
+                        const shardedData = result[key];
+                        for (const collectionName in shardedData) {
+                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
+                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
+                        }
+                    } else {
+                        standardResult[key] = result[key];
+                    }
+                }
+
+                // Add standard result to the batch
+                if (Object.keys(standardResult).length > 0) {
+                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                        .collection(config.resultsSubcollection).doc(mCalc.category)
+                        .collection(config.computationsSubcollection).doc(name);
+                    
+                    standardWrites.push({ ref: docRef, data: standardResult });
+                }
+                success++;
+            }
+        } catch (e) {
+            logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
+        }
+    }
+
+    // Commit standard (non-sharded) writes in chunks
+    if (standardWrites.length > 0) {
+        await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);
+    }
+
+    // Commit all sharded writes
+    for (const collectionName in shardedWrites) {
+        const docs = shardedWrites[collectionName];
+        const shardedDocWrites = [];
+        for (const docId in docs) {
+            // Special case for stateful meta-calcs that write to a specific path
+            const docRef = docId.includes('/')
+                ? deps.db.doc(docId) // docId is a full path
+                : deps.db.collection(collectionName).doc(docId); // docId is just an ID
+            
+            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
+        }
+        if (shardedDocWrites.length > 0) {
+            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
+        }
+    }
+    // --- END: FULL COMMIT LOGIC ---
+    
+    logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
-module.exports = { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass };
+
+module.exports = { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.129",
+  "version": "1.0.131",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [