bulltrackers-module 1.0.211 → 1.0.213

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,559 +1,543 @@
-/**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: 'commitResults' now isolates commits PER COMPUTATION.
- * A single failure (e.g., size limit) will only fail that specific calculation,
- * allowing others in the same pass/date to succeed and be recorded.
- */
-
-const { ComputationController }              = require('../controllers/computation_controller');
-const { batchStoreSchemas }                  = require('../utils/schema_capture');
-const { normalizeName, commitBatchInChunks } = require('../utils/utils');
-const {
-    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights,
-    getHistoryPartRefs, streamPortfolioData, streamHistoryData,
-    getRelevantShardRefs, loadDataByRefs
-} = require('../utils/data_loader');
-
-const {
-    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators,
-    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics,
-    TimeSeries, priceExtractor
-} = require('../layers/math_primitives.js');
-
-const pLimit = require('p-limit');
-
-/**
- * Groups calculations from a manifest by their 'pass' property.
- */
-function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
-
-/**
- * --- PASSIVE DATA VALIDATION ---
- */
-function validateResultPatterns(logger, calcName, results, category) {
-    if (category === 'speculator' || category === 'speculators') return;
-
-    const tickers    = Object.keys(results);
-    const totalItems = tickers.length;
-
-    if (totalItems < 5) return;
-
-    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
-    if (!sampleTicker) return;
-
-    const keys = Object.keys(results[sampleTicker]);
-
-    keys.forEach(key => {
-        if (key.startsWith('_')) return;
-
-        let nullCount      = 0;
-        let nanCount       = 0;
-        let undefinedCount = 0;
-
-        for (const t of tickers) {
-            const val = results[t][key];
-            if (val === null) nullCount++;
-            if (val === undefined) undefinedCount++;
-            if (typeof val === 'number' && isNaN(val)) nanCount++;
-        }
-
-        if (nanCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
-        } else if (undefinedCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`);
-        }
-        else if (nullCount > (totalItems * 0.9)) {
-            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
-        }
-    });
-}
-
-function checkRootDependencies(calcManifest, rootDataStatus) {
-    const missing = [];
-    if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
-    for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
-        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
-    }
-    return { canRun: missing.length === 0, missing };
-}
-
-async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger } = dependencies;
-    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData = null, socialData = null;
-
-    try {
-        const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs        (config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio  = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights           (config, dependencies, dateStr).then(r => { insightsData  = r; hasInsights   = !!r; }));
-        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
-        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
-
-        if (dateToProcess >= earliestDates.price) {
-            tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
-        }
-
-        await Promise.all(tasks);
-
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-
-        return {
-            portfolioRefs,
-            historyRefs,
-            todayInsights: insightsData,
-            todaySocialPostInsights: socialData,
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }
-        };
-
-    } catch (err) {
-        logger.log('ERROR', `Error checking data: ${err.message}`);
-        return null;
-    }
-}
-
-async function checkPriceDataAvailability(config, dependencies) {
-    const { db, logger } = dependencies;
-    const collection = config.priceCollection || 'asset_prices';
-    try {
-        const snapshot = await db.collection(collection).limit(1).get();
-        if (snapshot.empty) {
-            logger.log('WARN', `[checkPriceData] No price shards found in ${collection}`);
-            return false;
-        }
-        return true;
-    } catch (e) {
-        logger.log('ERROR', `[checkPriceData] Failed to check price availability: ${e.message}`);
-        return false;
-    }
-}
-
-async function fetchComputationStatus(dateStr, config, { db }) {
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc(dateStr);
-    const snap = await docRef.get();
-    return snap.exists ? snap.data() : {};
-}
-
-async function fetchGlobalComputationStatus(config, { db }) {
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc('global_status');
-    const snap       = await docRef.get();
-    return snap.exists ? snap.data() : {};
-}
-
-async function updateComputationStatus(dateStr, updates, config, { db }) {
-    if (!updates || Object.keys(updates).length === 0) return;
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc(dateStr);
-    await docRef.set(updates, { merge: true });
-}
-
-async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
-    if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc('global_status');
-    const flattenUpdates = {};
-    for (const [date, statuses] of Object.entries(updatesByDate)) {
-        for (const [calc, status] of Object.entries(statuses)) {
-            flattenUpdates[`${date}.${calc}`] = status;
-        }
-    }
-    try {
-        await docRef.update(flattenUpdates);
-    } catch (err) {
-        if (err.code === 5) {
-            const deepObj = {};
-            for (const [date, statuses] of Object.entries(updatesByDate)) {
-                deepObj[date] = statuses;
-            }
-            await docRef.set(deepObj, { merge: true });
-        } else {
-            throw err;
-        }
-    }
-}
-
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const calcsToFetch = new Set();
-    for (const calc of calcsInPass) {
-        if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
-        if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
-    }
-    if (!calcsToFetch.size) return {};
-    const fetched = {};
-    const docRefs = [];
-    const names   = [];
-    for (const name of calcsToFetch) {
-        const m = manifestMap.get(name);
-        if (m) {
-            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection).doc(name));
-            names.push(name);
-        }
-    }
-    if (docRefs.length) {
-        const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } });
-    }
-    return fetched;
-}
-
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
-    const { logger } = deps;
-    const controller     = new ComputationController(config, deps);
-    const calcs          = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c =>
-        c.manifest.rootDataDependencies.includes('portfolio') ||
-        c.manifest.rootDataDependencies.includes('history')
-    );
-
-    if (streamingCalcs.length === 0) return;
-
-    logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
-
-    await controller.loader.loadMappings();
-    const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-    const prevDateStr = prevDate.toISOString().slice(0, 10);
-
-    const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
-    const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-    const needsTradingHistory     = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter                 = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
-
-    let yP_chunk = {};
-    let tH_chunk = {};
-
-    for await (const tP_chunk of tP_iter) {
-        if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
-        if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-
-        const promises = streamingCalcs.map(calc =>
-            controller.executor.executePerUser(
-                calc,
-                calc.manifest,
-                dateStr,
-                tP_chunk,
-                yP_chunk,
-                tH_chunk,
-                fetchedDeps,
-                previousFetchedDeps
-            )
-        );
-        await Promise.all(promises);
-    }
-    logger.log('INFO', `[${passName}] Streaming complete.`);
-}
-
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
-    const dStr = date.toISOString().slice(0, 10);
-    const logger = deps.logger;
-    const fullRoot = { ...rootData };
-    if (calcs.some(c => c.isHistorical)) {
-        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
-        const prevStr = prev.toISOString().slice(0, 10);
-        fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
-    }
-
-    const state = {};
-    for (const c of calcs) {
-        try {
-            const inst = new c.class();
-            inst.manifest = c;
-            state[normalizeName(c.name)] = inst;
-            logger.log('INFO', `${c.name} calculation running for ${dStr}`);
-        }
-        catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
-    }
-
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
-    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
-}
-
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
-    const controller = new ComputationController(config, deps);
-    const dStr = date.toISOString().slice(0, 10);
-    const state = {};
-
-    for (const mCalc of calcs) {
-        try {
-            deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
-            const inst = new mCalc.class();
-            inst.manifest = mCalc;
-            await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
-            state[normalizeName(mCalc.name)] = inst;
-        } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
-    }
-    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
-}
-
-/**
- * --- REFACTORED: commitResults ---
- * Commits results individually per calculation.
- * If one calculation fails (e.g. size limit), others still succeed.
- */
-async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
-    const successUpdates = {};
-    const schemas = [];
-
-    // Iterate PER CALCULATION to isolate failures
-    for (const name in stateObj) {
-        const calc = stateObj[name];
-        let hasData = false;
-
-        try {
-            const result = await calc.getResult();
-            if (!result) {
-                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`);
-                continue;
-            }
-
-            const standardRes   = {};
-            const shardedWrites = [];
-            const calcWrites    = []; // Accumulate all writes for THIS specific calculation
-
-            // 1. Separate Standard and Sharded Data
-            for (const key in result) {
-                if (key.startsWith('sharded_')) {
-                    const sData = result[key];
-                    // sData structure: { CollectionName: { DocId: { ...data } } }
-                    for (const colName in sData) {
-                        const docsMap = sData[colName];
-                        for (const docId in docsMap) {
-                            // Support both full path or collection-relative path
-                            const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId);
-                            shardedWrites.push({
-                                ref,
-                                data: { ...docsMap[docId], _completed: true }
-                            });
-                        }
-                    }
-                    if (Object.keys(sData).length > 0) hasData = true;
-                } else {
-                    standardRes[key] = result[key];
-                }
-            }
-
-            // 2. Prepare Standard Result Write
-            if (Object.keys(standardRes).length) {
-                validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-                standardRes._completed = true;
-
-                const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                    .collection(config.computationsSubcollection).doc(name);
-
-                calcWrites.push({
-                    ref: docRef,
-                    data: standardRes
-                });
-                hasData = true;
-            }
-
-            // 3. Queue Schema (Safe to accumulate)
-            if (calc.manifest.class.getSchema) {
-                const { class: _cls, ...safeMetadata } = calc.manifest;
-                schemas.push({
-                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata
-                });
-            }
-
-            // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
-            if (hasData) {
-                // Combine standard + sharded writes for this unit of work
-                const allWritesForCalc = [...calcWrites, ...shardedWrites];
-
-                if (allWritesForCalc.length > 0) {
-                    await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
-
-                    // IF we get here, the commit succeeded.
-                    successUpdates[name] = true;
-                    deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
-                } else {
-                    deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`);
-                }
-            } else {
-                deps.logger.log('INFO', `${name} for ${dStr}: - Empty`);
-            }
-
-        } catch (e) {
-            // CRITICAL: Catch errors here so the loop continues for other calculations
-            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
-            // Do NOT add to successUpdates
-        }
-    }
-
-    // Save Schemas (Best effort, isolated)
-    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
-
-    // Update Status Document (Only for the ones that succeeded)
-    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
-        await updateComputationStatus(dStr, successUpdates, config, deps);
-        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
-    }
-    return successUpdates;
-}
-
-/**
- * --- UPDATED: runBatchPriceComputation ---
- */
-async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps;
-    const controller = new ComputationController(config, deps);
-
-    const mappings = await controller.loader.loadMappings();
-
-    let targetInstrumentIds = [];
-    if (targetTickers && targetTickers.length > 0) {
-        const tickerToInst = mappings.tickerToInstrument || {};
-        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        if (targetInstrumentIds.length === 0) {
-            logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
-            return;
-        }
-    }
-
-    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-
-    if (!allShardRefs.length) {
-        logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
-        return;
-    }
-
-    const OUTER_CONCURRENCY_LIMIT = 2;
-    const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50;
-
-    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
-
-    const shardChunks = [];
-    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
-        shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
-    }
-
-    const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
-
-    const chunkPromises = [];
-    for (let index = 0; index < shardChunks.length; index++) {
-        const shardChunkRefs = shardChunks[index];
-        chunkPromises.push(outerLimit(async () => {
-            try {
-                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-
-                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-
-                if (targetInstrumentIds.length > 0) {
-                    const requestedSet = new Set(targetInstrumentIds);
-                    for (const loadedInstrumentId in pricesData) {
-                        if (!requestedSet.has(loadedInstrumentId)) {
-                            delete pricesData[loadedInstrumentId];
-                        }
-                    }
-                }
-
-                // We now accumulate writes per calc to allow partial success, though batching optimization is tricky here.
-                // For safety, let's keep the existing structure but wrap individual calc processing in try/catch 
-                // inside the write phase if possible. 
-                // However, runBatchPrice is optimized for BULK throughput. 
-                // To prevent total failure, we will use a safe array.
-                const writes = [];
-
-                for (const dateStr of dateStrings) {
-                    const context = {
-                        mappings,
-                        prices: { history: pricesData },
-                        date: { today: dateStr },
-                        math: {
-                            extract: DataExtractor,
-                            history: HistoryExtractor,
-                            compute: MathPrimitives,
-                            aggregate: Aggregators,
-                            validate: Validators,
-                            signals: SignalPrimitives,
-                            schemas: SCHEMAS,
-                            distribution: DistributionAnalytics,
-                            TimeSeries: TimeSeries,
-                            priceExtractor: priceExtractor
-                        }
-                    };
-
-                    for (const calcManifest of calcs) {
-                        try {
-                            // logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`); // Verbose
-                            const instance = new calcManifest.class();
-                            await instance.process(context);
-                            const result = await instance.getResult();
-
-                            if (result && Object.keys(result).length > 0) {
-                                let dataToWrite = result;
-                                if (result.by_instrument) dataToWrite = result.by_instrument;
-
-                                if (Object.keys(dataToWrite).length > 0) {
-                                    const docRef = db.collection(config.resultsCollection).doc(dateStr)
-                                        .collection(config.resultsSubcollection).doc(calcManifest.category)
-                                        .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-
-                                    writes.push({
-                                        ref: docRef,
-                                        data: { ...dataToWrite, _completed: true },
-                                        options: { merge: true }
-                                    });
-                                }
-                            }
-                        } catch (err) {
-                            logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
-                        }
-                    }
-                }
-
-                if (writes.length > 0) {
-                    const commitBatches = [];
-                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
-                        commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
-                    }
-
-                    const commitLimit = pLimit(10);
-
-                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                        const batch = db.batch();
-                        batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-
-                        try {
-                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
-                        } catch (commitErr) {
-                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
-                        }
-                    })));
-                }
-
-            } catch (chunkErr) {
-                logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
-            }
-        }));
-    }
-
-    await Promise.all(chunkPromises);
-    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
-}
-
-module.exports = {
-    groupByPass,
-    checkRootDependencies,
-    checkRootDataAvailability,
-    fetchExistingResults,
-    fetchComputationStatus,
-    fetchGlobalComputationStatus,
-    updateComputationStatus,
-    updateGlobalComputationStatus,
-    runStandardComputationPass,
-    runMetaComputationPass,
-    runBatchPriceComputation
+/**
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
+ * FIXED: 'commitResults' now records the CODE HASH in the status document
+ * instead of a boolean, enabling auto-invalidation on code changes.
+ */
+
+const { ComputationController }              = require('../controllers/computation_controller');
+const { batchStoreSchemas }                  = require('../utils/schema_capture');
+const { normalizeName, commitBatchInChunks } = require('../utils/utils');
+const {
+    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights,
+    getHistoryPartRefs, streamPortfolioData, streamHistoryData,
+    getRelevantShardRefs, loadDataByRefs
+} = require('../utils/data_loader');
+
+const {
+    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators,
+    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics,
+    TimeSeries, priceExtractor
+} = require('../layers/math_primitives.js');
+
+const pLimit = require('p-limit');
+
+function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
+
+/**
+ * --- PASSIVE DATA VALIDATION ---
+ */
+function validateResultPatterns(logger, calcName, results, category) {
+    if (category === 'speculator' || category === 'speculators') return;
+
+    const tickers    = Object.keys(results);
+    const totalItems = tickers.length;
+
+    if (totalItems < 5) return;
+
+    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
+    if (!sampleTicker) return;
+
+    const keys = Object.keys(results[sampleTicker]);
+
+    keys.forEach(key => {
+        if (key.startsWith('_')) return;
+
+        let nullCount      = 0;
+        let nanCount       = 0;
+        let undefinedCount = 0;
+
+        for (const t of tickers) {
+            const val = results[t][key];
+            if (val === null) nullCount++;
+            if (val === undefined) undefinedCount++;
+            if (typeof val === 'number' && isNaN(val)) nanCount++;
+        }
+
+        if (nanCount === totalItems) {
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
+        } else if (undefinedCount === totalItems) {
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`);
+        }
+        else if (nullCount > (totalItems * 0.9)) {
+            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
+        }
+    });
+}
+
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    const missing = [];
+    if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
+    for (const dep of calcManifest.rootDataDependencies) {
+        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
+        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
+        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
+        else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
+    }
+    return { canRun: missing.length === 0, missing };
+}
+
+async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
+    const { logger } = dependencies;
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+    let portfolioRefs = [], historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData = null, socialData = null;
+
+    try {
+        const tasks = [];
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs        (config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio  = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights           (config, dependencies, dateStr).then(r => { insightsData  = r; hasInsights   = !!r; }));
+        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
+        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
+
+        if (dateToProcess >= earliestDates.price) {
+            tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
+        }
+
+        await Promise.all(tasks);
+
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
+
+        return {
+            portfolioRefs,
+            historyRefs,
+            todayInsights: insightsData,
+            todaySocialPostInsights: socialData,
+            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices },
+            yesterdayPortfolioRefs: null // Will be populated if needed
+        };
+
+    } catch (err) {
+        logger.log('ERROR', `Error checking data: ${err.message}`);
+        return null;
+    }
+}
+
+async function checkPriceDataAvailability(config, dependencies) {
+    const { db } = dependencies;
+    const collection = config.priceCollection || 'asset_prices';
+    try {
+        const snapshot = await db.collection(collection).limit(1).get();
+        if (snapshot.empty) return false;
+        return true;
+    } catch (e) {
+        return false;
+    }
+}
+
+async function fetchComputationStatus(dateStr, config, { db }) {
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc(dateStr);
+    const snap = await docRef.get();
+    return snap.exists ? snap.data() : {};
+}
+
+async function fetchGlobalComputationStatus(config, { db }) {
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef     = db.collection(collection).doc('global_status');
+    const snap       = await docRef.get();
+    return snap.exists ? snap.data() : {};
+}
+
+async function updateComputationStatus(dateStr, updates, config, { db }) {
+    if (!updates || Object.keys(updates).length === 0) return;
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef     = db.collection(collection).doc(dateStr);
+    await docRef.set(updates, { merge: true });
+}
+
+async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
+    if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef     = db.collection(collection).doc('global_status');
+    const flattenUpdates = {};
+    for (const [date, statuses] of Object.entries(updatesByDate)) {
+        for (const [calc, status] of Object.entries(statuses)) {
+            flattenUpdates[`${date}.${calc}`] = status;
+        }
+    }
+    try {
+        await docRef.update(flattenUpdates);
+    } catch (err) {
+        if (err.code === 5) {
+            const deepObj = {};
+            for (const [date, statuses] of Object.entries(updatesByDate)) {
+                deepObj[date] = statuses;
+            }
+            await docRef.set(deepObj, { merge: true });
+        } else {
+            throw err;
+        }
+    }
+}
+
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
+    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const calcsToFetch = new Set();
+    for (const calc of calcsInPass) {
+        if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
+        if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
+    }
+    if (!calcsToFetch.size) return {};
+    const fetched = {};
+    const docRefs = [];
+    const names   = [];
+    for (const name of calcsToFetch) {
+        const m = manifestMap.get(name);
+        if (m) {
+            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
+                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
+                .collection(config.computationsSubcollection).doc(name));
+            names.push(name);
+        }
+    }
+    if (docRefs.length) {
+        const snaps = await db.getAll(...docRefs);
+        snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } });
+    }
+    return fetched;
+}
+
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
+    const { logger } = deps;
+    const controller     = new ComputationController(config, deps);
+    const calcs          = Object.values(state).filter(c => c && c.manifest);
+    const streamingCalcs = calcs.filter(c =>
+        c.manifest.rootDataDependencies.includes('portfolio') ||
+        c.manifest.rootDataDependencies.includes('history')
+    );
+
+    if (streamingCalcs.length === 0) return;
+
+    logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
+
+    await controller.loader.loadMappings();
+    const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDateStr = prevDate.toISOString().slice(0, 10);
+
+    const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+    const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
+    const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    const needsTradingHistory     = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter                 = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
+
+    let yP_chunk = {};
+    let tH_chunk = {};
+
+    for await (const tP_chunk of tP_iter) {
+        if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
+        if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
+
+        const promises = streamingCalcs.map(calc =>
+            controller.executor.executePerUser(
+                calc,
+                calc.manifest,
+                dateStr,
+                tP_chunk,
+                yP_chunk,
+                tH_chunk,
+                fetchedDeps,
+                previousFetchedDeps
+            )
+        );
+        await Promise.all(promises);
+    }
+    logger.log('INFO', `[${passName}] Streaming complete.`);
+}
+
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
+    const dStr = date.toISOString().slice(0, 10);
+    const logger = deps.logger;
+    const fullRoot = { ...rootData };
+    if (calcs.some(c => c.isHistorical)) {
+        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+        const prevStr = prev.toISOString().slice(0, 10);
+        fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
+    }
+
+    const state = {};
+    for (const c of calcs) {
+        try {
+            const inst = new c.class();
+            inst.manifest = c;
+            state[normalizeName(c.name)] = inst;
+            logger.log('INFO', `${c.name} calculation running for ${dStr}`);
+        }
+        catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
+    }
+
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
+    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
+}
+
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
+    const controller = new ComputationController(config, deps);
+    const dStr = date.toISOString().slice(0, 10);
+    const state = {};
+
+    for (const mCalc of calcs) {
+        try {
+            deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
+            const inst = new mCalc.class();
+            inst.manifest = mCalc;
+            await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
+            state[normalizeName(mCalc.name)] = inst;
+        } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
+    }
+    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
+}
+
+/**
+ * --- REFACTORED: commitResults ---
+ * Commits results individually per calculation.
+ * If one calculation fails (e.g. size limit), others still succeed.
+ * UPDATED: Writes the HASH to the status document.
+ */
+async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
+    const successUpdates = {};
+    const schemas = [];
+
+    // Iterate PER CALCULATION to isolate failures
+    for (const name in stateObj) {
+        const calc = stateObj[name];
+        let hasData = false;
+
+        try {
+            const result = await calc.getResult();
+            if (!result) {
+                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`);
+                continue;
+            }
+
+            const standardRes   = {};
+            const shardedWrites = [];
+            const calcWrites    = []; 
+
+            // 1. Separate Standard and Sharded Data
+            for (const key in result) {
+                if (key.startsWith('sharded_')) {
+                    const sData = result[key];
+                    for (const colName in sData) {
+                        const docsMap = sData[colName];
+                        for (const docId in docsMap) {
+                            const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId);
+                            shardedWrites.push({
+                                ref,
+                                data: { ...docsMap[docId], _completed: true }
+                            });
+                        }
+                    }
+                    if (Object.keys(sData).length > 0) hasData = true;
+                } else {
+                    standardRes[key] = result[key];
+                }
+            }
+
+            // 2. Prepare Standard Result Write
+            if (Object.keys(standardRes).length) {
+                validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
+                standardRes._completed = true;
+
+                const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
+                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
+                    .collection(config.computationsSubcollection).doc(name);
+
+                calcWrites.push({
+                    ref: docRef,
+                    data: standardRes
+                });
+                hasData = true;
+            }
+
+            // 3. Queue Schema (Safe to accumulate)
+            if (calc.manifest.class.getSchema) {
+                const { class: _cls, ...safeMetadata } = calc.manifest;
+                schemas.push({
+                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata
+                });
+            }
+
+            // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
+            if (hasData) {
+                const allWritesForCalc = [...calcWrites, ...shardedWrites];
+
+                if (allWritesForCalc.length > 0) {
+                    await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
+
+                    // --- CRITICAL UPDATE: Store the Smart Hash ---
+                    successUpdates[name] = calc.manifest.hash || true;
+                    
+                    deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
+                } else {
+                    deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`);
+                }
+            } else {
+                deps.logger.log('INFO', `${name} for ${dStr}: - Empty`);
+            }
+
+        } catch (e) {
+            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
+        }
+    }
+
+    // Save Schemas (Best effort, isolated)
+    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
+
+    // Update Status Document (Only for the ones that succeeded)
+    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
+        await updateComputationStatus(dStr, successUpdates, config, deps);
+        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
+    }
+    return successUpdates;
+}
+
+/**
+ * --- UPDATED: runBatchPriceComputation ---
+ */
+async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
+    const { logger, db, calculationUtils } = deps;
+    const controller = new ComputationController(config, deps);
+
+    const mappings = await controller.loader.loadMappings();
+
+    let targetInstrumentIds = [];
+    if (targetTickers && targetTickers.length > 0) {
+        const tickerToInst = mappings.tickerToInstrument || {};
+        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
+        if (targetInstrumentIds.length === 0) {
+            logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
+            return;
+        }
+    }
+
+    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
+
+    if (!allShardRefs.length) {
+        logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
+        return;
+    }
+
+    const OUTER_CONCURRENCY_LIMIT = 2;
+    const SHARD_BATCH_SIZE = 20;
+    const WRITE_BATCH_LIMIT = 50;
+
+    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
+
+    const shardChunks = [];
+    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
+        shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
+    }
+
+    const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
+
+    const chunkPromises = [];
+    for (let index = 0; index < shardChunks.length; index++) {
+        const shardChunkRefs = shardChunks[index];
+        chunkPromises.push(outerLimit(async () => {
+            try {
+                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
+
+                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
+
+                if (targetInstrumentIds.length > 0) {
+                    const requestedSet = new Set(targetInstrumentIds);
+                    for (const loadedInstrumentId in pricesData) {
+                        if (!requestedSet.has(loadedInstrumentId)) {
+                            delete pricesData[loadedInstrumentId];
+                        }
+                    }
+                }
+
+                const writes = [];
+
+                for (const dateStr of dateStrings) {
+                    const context = {
+                        mappings,
+                        prices: { history: pricesData },
+                        date: { today: dateStr },
+                        math: {
+                            extract: DataExtractor,
+                            history: HistoryExtractor,
+                            compute: MathPrimitives,
+                            aggregate: Aggregators,
+                            validate: Validators,
+                            signals: SignalPrimitives,
+                            schemas: SCHEMAS,
+                            distribution: DistributionAnalytics,
+                            TimeSeries: TimeSeries,
+                            priceExtractor: priceExtractor
+                        }
+                    };
+
+                    for (const calcManifest of calcs) {
+                        try {
+                            const instance = new calcManifest.class();
+                            await instance.process(context);
+                            const result = await instance.getResult();
+
+                            if (result && Object.keys(result).length > 0) {
+                                let dataToWrite = result;
+                                if (result.by_instrument) dataToWrite = result.by_instrument;
+
+                                if (Object.keys(dataToWrite).length > 0) {
+                                    const docRef = db.collection(config.resultsCollection).doc(dateStr)
+                                        .collection(config.resultsSubcollection).doc(calcManifest.category)
+                                        .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+
+                                    writes.push({
+                                        ref: docRef,
+                                        data: { ...dataToWrite, _completed: true },
+                                        options: { merge: true }
+                                    });
+                                }
+                            }
+                        } catch (err) {
+                            logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
+                        }
+                    }
+                }
+
+                if (writes.length > 0) {
+                    const commitBatches = [];
+                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
+                        commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
+                    }
+
+                    const commitLimit = pLimit(10);
+
+                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
+                        const batch = db.batch();
+                        batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+
+                        try {
+                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
+                        } catch (commitErr) {
+                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
+                        }
+                    })));
+                }
+
+            } catch (chunkErr) {
+                logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
+            }
+        }));
+    }
+
+    await Promise.all(chunkPromises);
+    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
+}
+
+module.exports = {
+    groupByPass,
+    checkRootDependencies,
+    checkRootDataAvailability,
+    fetchExistingResults,
+    fetchComputationStatus,
+    fetchGlobalComputationStatus,
+    updateComputationStatus,
+    updateGlobalComputationStatus,
+    runStandardComputationPass,
+    runMetaComputationPass,
+    runBatchPriceComputation
 };