bulltrackers-module 1.0.219 → 1.0.220

package/functions/computation-system/data/DependencyFetcher.js

@@ -0,0 +1,70 @@
+/**
+ * @fileoverview Fetches results from previous computations, handling auto-sharding hydration.
+ */
+const { normalizeName } = require('../utils/utils');
+
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
+    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const calcsToFetch = new Set();
+    
+    for (const calc of calcsInPass) { 
+        if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); 
+        if (includeSelf && calc.isHistorical) calcsToFetch.add(normalizeName(calc.name)); 
+    }
+    
+    if (!calcsToFetch.size) return {};
+    
+    const fetched = {};
+    const docRefs = [];
+    const names = [];
+
+    for (const name of calcsToFetch) { 
+        const m = manifestMap.get(name); 
+        if (m) { 
+            docRefs.push(db.collection(config.resultsCollection)
+                .doc(dateStr)
+                .collection(config.resultsSubcollection)
+                .doc(m.category || 'unknown')
+                .collection(config.computationsSubcollection)
+                .doc(name)); 
+            names.push(name); 
+        } 
+    }
+
+    if (docRefs.length) { 
+        const snaps = await db.getAll(...docRefs); 
+        const hydrationPromises = [];
+        
+        snaps.forEach((doc, i) => { 
+            const name = names[i]; 
+            if (!doc.exists) return; 
+            const data = doc.data(); 
+            if (data._sharded === true) { 
+                hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); 
+            } else if (data._completed) { 
+                fetched[name] = data; 
+            } 
+        });
+
+        if (hydrationPromises.length > 0) { 
+            const hydratedResults = await Promise.all(hydrationPromises); 
+            hydratedResults.forEach(res => { fetched[res.name] = res.data; }); 
+        } 
+    }
+    return fetched;
+}
+
+async function hydrateAutoShardedResult(docRef, resultName) {
+    const shardsCol = docRef.collection('_shards');
+    const snapshot = await shardsCol.get();
+    const assembledData = { _completed: true }; 
+    snapshot.forEach(doc => { 
+        const chunk = doc.data(); 
+        Object.assign(assembledData, chunk); 
+    });
+    delete assembledData._sharded;
+    delete assembledData._completed;
+    return { name: resultName, data: assembledData };
+}
+
+module.exports = { fetchExistingResults };

package/functions/computation-system/executors/MetaExecutor.js

@@ -0,0 +1,68 @@
+/**
+ * @fileoverview Executor for "Meta" (global) calculations.
+ */
+const { normalizeName } = require('../utils/utils');
+const { CachedDataLoader } = require('../data/CachedDataLoader');
+const { ContextFactory } = require('../context/ContextFactory');
+const { commitResults } = require('../persistence/ResultCommitter');
+
+class MetaExecutor {
+    static async run(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
+        const dStr = date.toISOString().slice(0, 10);
+        const state = {};
+        const cachedLoader = new CachedDataLoader(config, deps);
+
+        for (const mCalc of calcs) {
+            try {
+                deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
+                const inst = new mCalc.class(); 
+                inst.manifest = mCalc;
+                
+                await MetaExecutor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps, config, deps, cachedLoader);
+                state[normalizeName(mCalc.name)] = inst;
+            } catch (e) { 
+                deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); 
+            }
+        }
+        return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
+    }
+
+    static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
+        const mappings = await loader.loadMappings();
+        const { logger } = deps;
+        const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
+        const social = metadata.rootDataDependencies?.includes('social') ? { today: await loader.loadSocial(dateStr) } : null;
+        
+        if (metadata.rootDataDependencies?.includes('price')) {
+            logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
+            const shardRefs = await loader.getPriceShardReferences();
+            if (shardRefs.length === 0) { logger.log('WARN', '[Executor] No price shards found.'); return {}; }
+            
+            let processedCount = 0;
+            for (const ref of shardRefs) {
+                const shardData = await loader.loadPriceShard(ref);
+                const partialContext = ContextFactory.buildMetaContext({ 
+                    dateStr, metadata, mappings, insights, socialData: social, 
+                    prices: { history: shardData }, computedDependencies: computedDeps, 
+                    previousComputedDependencies: prevDeps, config, deps 
+                });
+                
+                await calcInstance.process(partialContext);
+                partialContext.prices = null; 
+                processedCount++;
+                if (processedCount % 10 === 0 && global.gc) { global.gc(); }
+            }
+            logger.log('INFO', `[Executor] Finished Batched Execution for ${metadata.name} (${processedCount} shards).`);
+            return calcInstance.getResult ? await calcInstance.getResult() : {};
+        } else {
+            const context = ContextFactory.buildMetaContext({ 
+                dateStr, metadata, mappings, insights, socialData: social, 
+                prices: {}, computedDependencies: computedDeps, 
+                previousComputedDependencies: prevDeps, config, deps 
+            });
+            return await calcInstance.process(context);
+        }
+    }
+}
+
+module.exports = { MetaExecutor };

package/functions/computation-system/executors/PriceBatchExecutor.js

@@ -0,0 +1,99 @@
+/**
+ * @fileoverview Specialized Executor for Price-Dependent Batch computations.
+ */
+const pLimit = require('p-limit');
+const { normalizeName } = require('../utils/utils');
+const { getRelevantShardRefs, loadDataByRefs } = require('../utils/data_loader');
+const { CachedDataLoader } = require('../data/CachedDataLoader');
+const mathLayer = require('../layers/index');
+const { LEGACY_MAPPING } = require('../topology/HashManager');
+
+async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
+    const { logger, db, calculationUtils } = deps;
+    const cachedLoader = new CachedDataLoader(config, deps);
+    const mappings = await cachedLoader.loadMappings();
+    
+    let targetInstrumentIds = [];
+    if (targetTickers && targetTickers.length > 0) {
+        const tickerToInst = mappings.tickerToInstrument || {};
+        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
+        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } 
+    }
+    
+    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
+    if (!allShardRefs.length) { logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.'); return; }
+    
+    const OUTER_CONCURRENCY_LIMIT = 2, SHARD_BATCH_SIZE = 20, WRITE_BATCH_LIMIT = 50;
+    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
+    
+    const shardChunks = []; 
+    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
+    
+    const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
+    const chunkPromises = [];
+
+    for (let index = 0; index < shardChunks.length; index++) {
+        const shardChunkRefs = shardChunks[index];
+        chunkPromises.push(outerLimit(async () => {
+            try {
+                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
+                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+                if (targetInstrumentIds.length > 0) { 
+                    const requestedSet = new Set(targetInstrumentIds); 
+                    for (const loadedInstrumentId in pricesData) { 
+                        if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } 
+                    } 
+                }
+                
+                const writes = [];
+                for (const dateStr of dateStrings) {
+                    const dynamicMathContext = {};
+                    for (const [key, value] of Object.entries(mathLayer)) { 
+                        dynamicMathContext[key] = value; 
+                        if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} 
+                    }
+                    const context = { mappings, prices: { history: pricesData }, date: { today: dateStr }, math: dynamicMathContext };
+                    
+                    for (const calcManifest of calcs) {
+                        try {
+                            const instance = new calcManifest.class(); 
+                            await instance.process(context); 
+                            const result = await instance.getResult();
+                            if (result && Object.keys(result).length > 0) {
+                                let dataToWrite = result; 
+                                if (result.by_instrument) dataToWrite = result.by_instrument;
+                                if (Object.keys(dataToWrite).length > 0) {
+                                    const docRef = db.collection(config.resultsCollection)
+                                        .doc(dateStr)
+                                        .collection(config.resultsSubcollection)
+                                        .doc(calcManifest.category)
+                                        .collection(config.computationsSubcollection)
+                                        .doc(normalizeName(calcManifest.name));
+                                    writes.push({ ref: docRef, data: { ...dataToWrite, _completed: true }, options: { merge: true } });
+                                }
+                            }
+                        } catch (err) { logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`); }
+                    }
+                }
+                
+                if (writes.length > 0) {
+                    const commitBatches = []; 
+                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
+                    const commitLimit = pLimit(10);
+                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
+                        const batch = db.batch(); batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+                        try { 
+                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`); 
+                        } catch (commitErr) { 
+                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); 
+                        }
+                    })));
+                }
+            } catch (chunkErr) { logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message }); }
+        }));
+    }
+    await Promise.all(chunkPromises);
+    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
+}
+
+module.exports = { runBatchPriceComputation };

package/functions/computation-system/executors/StandardExecutor.js

@@ -0,0 +1,115 @@
+/**
+ * @fileoverview Executor for "Standard" (per-user) calculations.
+ */
+const { normalizeName } = require('../utils/utils');
+const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
+const { CachedDataLoader } = require('../data/CachedDataLoader');
+const { ContextFactory } = require('../context/ContextFactory');
+const { commitResults } = require('../persistence/ResultCommitter');
+const mathLayer = require('../layers/index');
+
+class StandardExecutor {
+    static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
+        const dStr = date.toISOString().slice(0, 10);
+        const logger = deps.logger;
+        
+        // 1. Prepare Yesterdays Data if needed
+        const fullRoot = { ...rootData };
+        if (calcs.some(c => c.isHistorical)) {
+            const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+            const prevStr = prev.toISOString().slice(0, 10);
+            fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
+        }
+
+        // 2. Initialize Instances
+        const state = {};
+        for (const c of calcs) { 
+            try { 
+                const inst = new c.class(); 
+                inst.manifest = c; 
+                state[normalizeName(c.name)] = inst; 
+                logger.log('INFO', `${c.name} calculation running for ${dStr}`); 
+            } catch (e) { 
+                logger.log('WARN', `Failed to init ${c.name}`); 
+            } 
+        }
+
+        // 3. Stream & Process
+        await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
+        
+        // 4. Commit
+        return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
+    }
+
+    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
+        const { logger } = deps;
+        const calcs = Object.values(state).filter(c => c && c.manifest);
+        const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
+        
+        if (streamingCalcs.length === 0) return;
+        
+        logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
+        
+        const cachedLoader = new CachedDataLoader(config, deps);
+        await cachedLoader.loadMappings();
+        
+        const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        const prevDateStr = prevDate.toISOString().slice(0, 10);
+        
+        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+        const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
+        const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) 
+            ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) 
+            : null;
+        
+        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+        const tH_iter = (needsTradingHistory && historyRefs) 
+            ? streamHistoryData(config, deps, dateStr, historyRefs) 
+            : null;
+
+        let yP_chunk = {}, tH_chunk = {};
+        
+        for await (const tP_chunk of tP_iter) {
+            if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
+            if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
+            
+            // Execute chunk for all calcs
+            const promises = streamingCalcs.map(calc => 
+                StandardExecutor.executePerUser(calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, fetchedDeps, previousFetchedDeps, config, deps, cachedLoader)
+            );
+            await Promise.all(promises);
+        }
+        logger.log('INFO', `[${passName}] Streaming complete.`);
+    }
+
+    static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader) {
+        const { logger } = deps;
+        const targetUserType = metadata.userType; 
+        const mappings = await loader.loadMappings();
+        const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
+        const SCHEMAS = mathLayer.SCHEMAS;
+
+        for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
+            const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
+            const todayHistory = historyData ? historyData[userId] : null;
+            const actualUserType = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
+            
+            if (targetUserType !== 'all') {
+                 const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
+                 if (mappedTarget !== actualUserType) continue;
+            }
+            
+            const context = ContextFactory.buildPerUserContext({ 
+                todayPortfolio, yesterdayPortfolio, todayHistory, userId, 
+                userType: actualUserType, dateStr, metadata, mappings, insights, 
+                computedDependencies: computedDeps, previousComputedDependencies: prevDeps, 
+                config, deps 
+            });
+            
+            try { await calcInstance.process(context); } 
+            catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
+        }
+    }
+}
+
+module.exports = { StandardExecutor };

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,12 +1,11 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Dispatches computation tasks to Pub/Sub for scalable execution.
- * FIXED: Instantiates PubSubUtils locally to ensure valid logger/dependencies are used.
- * IMPROVED: Logging now explicitly lists the calculations being scheduled.
+ * REFACTORED: Now uses WorkflowOrchestrator for helper functions.
  */
 
 const { getExpectedDateStrings } = require('../utils/utils.js');
-const { groupByPass } = require('./orchestration_helpers.js');
+const { groupByPass } = require('../WorkflowOrchestrator.js');
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 
 const TOPIC_NAME = 'computation-tasks';
@@ -19,6 +18,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const { logger } = dependencies;
     const pubsubUtils = new PubSubUtils(dependencies);
     const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    
     if (!passToRun) { return logger.log('ERROR', '[Dispatcher] No pass defined (COMPUTATION_PASS_TO_RUN). Aborting.'); }
 
     // 1. Validate Pass Existence

package/functions/computation-system/helpers/computation_worker.js

@@ -1,11 +1,10 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * FIXED: Added robust payload parsing to handle Cloud Functions Gen 2 (CloudEvents).
+ * REFACTORED: Now imports logic from the new WorkflowOrchestrator.
  */
 
-const { runDateComputation } = require('./computation_pass_runner.js');
-const { groupByPass } = require('./orchestration_helpers.js');
+const { runDateComputation, groupByPass } = require('../WorkflowOrchestrator.js');
 
 /**
  * Handles a single Pub/Sub message for a computation task.
@@ -16,29 +15,56 @@ async function handleComputationTask(message, config, dependencies, computationM
 
     let data;
     try {
-        // 1. Handle Cloud Functions Gen 2 (CloudEvent)
-        if (message.data && message.data.message && message.data.message.data) { const buffer = Buffer.from(message.data.message.data, 'base64'); data = JSON.parse(buffer.toString()); }
-        // 2. Handle Cloud Functions Gen 1 / Legacy PubSub
-        else if (message.data && typeof message.data === 'string') { const buffer = Buffer.from(message.data, 'base64'); data = JSON.parse(buffer.toString()); }
-        // 3. Handle Direct JSON (Test harness or simulator)
-        else if (message.json) { data = message.json; }
-        // 4. Fallback: Assume message is the payload
-        else { data = message; }
-    } catch (parseError) { logger.log('ERROR', `[Worker] Failed to parse Pub/Sub payload.`, { error: parseError.message }); return; }
+        // 1. Handle Cloud Functions Gen 2 (CloudEvent) -> Gen 1 -> Direct JSON -> Message
+        if (message.data && message.data.message && message.data.message.data) { 
+            const buffer = Buffer.from(message.data.message.data, 'base64'); data = JSON.parse(buffer.toString()); 
+        } else if (message.data && typeof message.data === 'string') { 
+            const buffer = Buffer.from(message.data, 'base64'); data = JSON.parse(buffer.toString()); 
+        } else if (message.json) { 
+            data = message.json; 
+        } else { 
+            data = message; 
+        }
+    } catch (parseError) { 
+        logger.log('ERROR', `[Worker] Failed to parse Pub/Sub payload.`, { error: parseError.message }); 
+        return; 
+    }
 
     try {
         // Validate Action
-        if (!data || data.action !== 'RUN_COMPUTATION_DATE') { if (data) logger.log('WARN', `[Worker] Unknown or missing action: ${data?.action}. Ignoring.`); return; }
+        if (!data || data.action !== 'RUN_COMPUTATION_DATE') { 
+            if (data) logger.log('WARN', `[Worker] Unknown or missing action: ${data?.action}. Ignoring.`); 
+            return; 
+        }
+        
         const { date, pass } = data;
-        if (!date || !pass) { logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`); return; }
+        if (!date || !pass) { 
+            logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`); 
+            return; 
+        }
+        
         logger.log('INFO', `[Worker] Received task: Date=${date}, Pass=${pass}`);
+        
         const passes = groupByPass(computationManifest);
         const calcsInThisPass = passes[pass] || [];
-        if (!calcsInThisPass.length) { logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`); return; }
+        
+        if (!calcsInThisPass.length) { 
+            logger.log('WARN', `[Worker] No calculations found for Pass ${pass}.`); 
+            return; 
+        }
+        
         const result = await runDateComputation(date, pass, calcsInThisPass, config, dependencies, computationManifest);
-        if (result) { logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates || {}).length}`);
-        } else { logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - Skipped (Dependencies missing or already done).`); }
-    } catch (err) { logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack }); throw err;  }
+        
+        if (result) { 
+            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates || {}).length}`);
+        } else { 
+            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - Skipped (Dependencies missing or already done).`); 
+        }
+        
+    } catch (err) { 
+        logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack }); 
+        throw err;  
+    }
 }
 
 module.exports = { handleComputationTask };

package/functions/computation-system/layers/mathematics.js

@@ -59,7 +59,7 @@ class MathPrimitives {
              return 1.0;
         }
 
-        const probability = normCDF(( -Math.abs(b) - nu * t ) / sigmaSqrtT) +  Math.exp((2 * nu * Math.abs(b)) / (sigma * sigma)) * normCDF(( -Math.abs(b) + nu * t ) / sigmaSqrtT);
+        const probability = normCDF(-term3) + Math.exp(term2) * normCDF(-term1);
 
         return Math.min(Math.max(probability, 0), 1);
     }

package/functions/computation-system/persistence/FirestoreUtils.js

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview Low-level Firestore interactions.
+ */
+const { withRetry } = require('../utils/utils.js'); // Assuming this exists or is passed in deps
+
+async function commitBatchInChunks(config, deps, writes, operationName) {
+    const { db, logger, calculationUtils } = deps;
+    const retryFn = calculationUtils ? calculationUtils.withRetry : (fn) => fn();
+
+    if (!writes || !writes.length) {
+        logger.log('WARN', `[${operationName}] No writes to commit.`);
+        return;
+    }
+
+    const MAX_BATCH_OPS = 300;
+    const MAX_BATCH_BYTES = 9 * 1024 * 1024; 
+
+    let currentBatch = db.batch();
+    let currentOpsCount = 0;
+    let currentBytesEst = 0;
+    let batchIndex = 1;
+
+    const commitAndReset = async () => {
+        if (currentOpsCount > 0) {
+            try {
+                await retryFn(
+                    () => currentBatch.commit(),
+                    `${operationName} (Chunk ${batchIndex})`
+                );
+                logger.log('INFO', `[${operationName}] Committed chunk ${batchIndex} (${currentOpsCount} ops, ~${(currentBytesEst / 1024 / 1024).toFixed(2)} MB).`);
+                batchIndex++;
+            } catch (err) {
+                logger.log('ERROR', `[${operationName}] Failed to commit chunk ${batchIndex}. Size: ${(currentBytesEst / 1024 / 1024).toFixed(2)} MB.`, { error: err.message });
+                throw err;
+            }
+        }
+        currentBatch = db.batch();
+        currentOpsCount = 0;
+        currentBytesEst = 0;
+    };
+
+    for (const write of writes) {
+        let docSize = 100;
+        try { if (write.data) docSize = JSON.stringify(write.data).length; } catch (e) { }
+
+        if (docSize > 900 * 1024) {
+            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB).`);
+        }
+
+        if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) {
+            await commitAndReset();
+        }
+
+        const options = write.options || { merge: true };
+        currentBatch.set(write.ref, write.data, options);
+        
+        currentOpsCount++;
+        currentBytesEst += docSize;
+    }
+
+    await commitAndReset();
+}
+
+module.exports = { commitBatchInChunks };

package/functions/computation-system/persistence/ResultCommitter.js

@@ -0,0 +1,118 @@
+/**
+ * @fileoverview Handles saving computation results with transparent auto-sharding.
+ */
+const { commitBatchInChunks } = require('./FirestoreUtils');
+const { updateComputationStatus } = require('./StatusRepository');
+const { batchStoreSchemas } = require('../utils/schema_capture');
+
+async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
+    const successUpdates = {};
+    const schemas = [];
+
+    for (const name in stateObj) {
+        const calc = stateObj[name];
+        try {
+            const result = await calc.getResult();
+            if (!result) { 
+                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`); 
+                continue; 
+            }
+
+            const mainDocRef = deps.db.collection(config.resultsCollection)
+                .doc(dStr)
+                .collection(config.resultsSubcollection)
+                .doc(calc.manifest.category)
+                .collection(config.computationsSubcollection)
+                .doc(name);
+
+            const updates = await prepareAutoShardedWrites(result, mainDocRef, deps.logger);
+
+            if (calc.manifest.class.getSchema) {
+                const { class: _cls, ...safeMetadata } = calc.manifest;
+                schemas.push({ 
+                    name, 
+                    category: calc.manifest.category, 
+                    schema: calc.manifest.class.getSchema(), 
+                    metadata: safeMetadata 
+                });
+            }
+
+            if (updates.length > 0) {
+                await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                successUpdates[name] = calc.manifest.hash || true;
+                const isSharded = updates.some(u => u.data._sharded === true);
+                deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written ${isSharded ? 'Sharded' : 'Standard'})`);
+            } else {
+                deps.logger.log('INFO', `${name} for ${dStr}: - Empty Data`);
+            }
+        } catch (e) {
+            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
+        }
+    }
+
+    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => {});
+
+    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
+        await updateComputationStatus(dStr, successUpdates, config, deps);
+        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
+    }
+    return successUpdates;
+}
+
+function calculateFirestoreBytes(value) {
+    if (value === null) return 1;
+    if (value === undefined) return 0;
+    if (typeof value === 'boolean') return 1;
+    if (typeof value === 'number') return 8;
+    if (typeof value === 'string') return Buffer.byteLength(value, 'utf8') + 1;
+    if (value instanceof Date) return 8;
+    if (value.constructor && value.constructor.name === 'DocumentReference') { return Buffer.byteLength(value.path, 'utf8') + 16; }
+    if (Array.isArray(value)) { let sum = 0; for (const item of value) sum += calculateFirestoreBytes(item); return sum; }
+    if (typeof value === 'object') { let sum = 0; for (const k in value) { if (Object.prototype.hasOwnProperty.call(value, k)) { sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]); } } return sum; }
+    return 0;
+}
+
+async function prepareAutoShardedWrites(result, docRef, logger) {
+    const SAFETY_THRESHOLD_BYTES = 1000 * 1024; // 1MB Limit
+    const OVERHEAD_ALLOWANCE = 20 * 1024;
+    const CHUNK_LIMIT = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
+    const totalSize = calculateFirestoreBytes(result);
+    const docPathSize = Buffer.byteLength(docRef.path, 'utf8') + 16;
+
+    if ((totalSize + docPathSize) < CHUNK_LIMIT) { 
+        const data = { ...result, _completed: true, _sharded: false }; 
+        return [{ ref: docRef, data, options: { merge: true } }]; 
+    }
+
+    logger.log('INFO', `[AutoShard] Result size ~${Math.round(totalSize/1024)}KB exceeds limit. Sharding...`);
+    const writes = [];
+    const shardCollection = docRef.collection('_shards');
+    let currentChunk = {};
+    let currentChunkSize = 0;
+    let shardIndex = 0;
+
+    for (const [key, value] of Object.entries(result)) { 
+        if (key.startsWith('_')) continue;
+        const keySize = Buffer.byteLength(key, 'utf8') + 1;         
+        const valueSize = calculateFirestoreBytes(value);
+        const itemSize = keySize + valueSize;
+
+        if (currentChunkSize + itemSize > CHUNK_LIMIT) { 
+            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } });
+            shardIndex++;
+            currentChunk = {};
+            currentChunkSize = 0;
+        }
+        currentChunk[key] = value;
+        currentChunkSize += itemSize;
+    }
+    if (Object.keys(currentChunk).length > 0) { 
+        writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); 
+    }
+    
+    const pointerData = { _completed: true, _sharded: true, _shardCount: shardIndex + 1, _lastUpdated: new Date().toISOString() };
+    writes.push({ ref: docRef, data: pointerData, options: { merge: false } });
+    return writes;
+}
+
+module.exports = { commitResults };