bulltrackers-module 1.0.214 → 1.0.216

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,7 +1,13 @@
 /**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: 'commitResults' now records the CODE HASH in the status document
- * instead of a boolean, enabling auto-invalidation on code changes.
+ * FILENAME: computation-system/helpers/orchestration_helpers.js
+ * FEATURE: Dynamic Auto-Sharding (Transparent 1MB Limit Handling)
+ * * DESCRIPTION:
+ * This module orchestrates the execution of computations. It handles:
+ * 1. Data Availability Checks
+ * 2. Dependency Injection (fetching results from previous passes)
+ * 3. Transparent Auto-Sharding:
+ * - Writes: Automatically detects if a result > 900KB. Splits it into a '_shards' subcollection.
+ * - Reads: Automatically detects sharded pointers and re-assembles the data.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -12,55 +18,27 @@ const {
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
-
-const {
-    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators,
-    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics,
-    TimeSeries, priceExtractor
-} = require('../layers/math_primitives.js');
-
+const mathLayer = require('../layers/index.js');
 const pLimit = require('p-limit');
 
+// Mappings for backward compatibility
+const LEGACY_MAPPING = {
+    DataExtractor: 'extract', HistoryExtractor: 'history', MathPrimitives: 'compute', Aggregators: 'aggregate', Validators: 'validate', SignalPrimitives: 'signals', SCHEMAS: 'schemas', DistributionAnalytics: 'distribution', TimeSeries: 'TimeSeries', priceExtractor: 'priceExtractor', InsightsExtractor: 'insights', UserClassifier: 'classifier', CognitiveBiases: 'bias', SkillAttribution: 'skill', Psychometrics: 'psychometrics'
+};
+
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
-/**
- * --- PASSIVE DATA VALIDATION ---
- */
 function validateResultPatterns(logger, calcName, results, category) {
     if (category === 'speculator' || category === 'speculators') return;
-
-    const tickers    = Object.keys(results);
-    const totalItems = tickers.length;
-
-    if (totalItems < 5) return;
-
-    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
-    if (!sampleTicker) return;
-
-    const keys = Object.keys(results[sampleTicker]);
-
-    keys.forEach(key => {
+    const tickers = Object.keys(results); const totalItems = tickers.length; if (totalItems < 5) return;
+    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object'); if (!sampleTicker) return;
+    Object.keys(results[sampleTicker]).forEach(key => {
         if (key.startsWith('_')) return;
-
-        let nullCount      = 0;
-        let nanCount       = 0;
-        let undefinedCount = 0;
-
-        for (const t of tickers) {
-            const val = results[t][key];
-            if (val === null) nullCount++;
-            if (val === undefined) undefinedCount++;
-            if (typeof val === 'number' && isNaN(val)) nanCount++;
-        }
-
-        if (nanCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
-        } else if (undefinedCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`);
-        }
-        else if (nullCount > (totalItems * 0.9)) {
-            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
-        }
+        let nullCount = 0, nanCount = 0, undefinedCount = 0;
+        for (const t of tickers) { const val = results[t][key]; if (val === null) nullCount++; if (val === undefined) undefinedCount++; if (typeof val === 'number' && isNaN(val)) nanCount++; }
+        if (nanCount === totalItems) logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of items.`);
+        else if (undefinedCount === totalItems) logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of items.`);
+        else if (nullCount > (totalItems * 0.9)) logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
     });
 }
 
@@ -68,11 +46,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
-        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
+        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+        else if (dep === 'price' && !rootDataStatus.hasPrices) missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -81,166 +59,112 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData = null, socialData = null;
-
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false, insightsData = null, socialData = null;
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs        (config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio  = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights           (config, dependencies, dateStr).then(r => { insightsData  = r; hasInsights   = !!r; }));
-        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
-        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
-
-        if (dateToProcess >= earliestDates.price) {
-            tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
-        }
-
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        if (dateToProcess >= earliestDates.price) { tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; })); }
         await Promise.all(tasks);
-
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-
-        return {
-            portfolioRefs,
-            historyRefs,
-            todayInsights: insightsData,
-            todaySocialPostInsights: socialData,
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices },
-            yesterdayPortfolioRefs: null // Will be populated if needed
-        };
-
-    } catch (err) {
-        logger.log('ERROR', `Error checking data: ${err.message}`);
-        return null;
-    }
+        return { portfolioRefs, historyRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }, yesterdayPortfolioRefs: null };
+    } catch (err) { logger.log('ERROR', `Error checking data: ${err.message}`); return null; }
 }
 
-async function checkPriceDataAvailability(config, dependencies) {
-    const { db } = dependencies;
-    const collection = config.priceCollection || 'asset_prices';
-    try {
-        const snapshot = await db.collection(collection).limit(1).get();
-        if (snapshot.empty) return false;
-        return true;
-    } catch (e) {
-        return false;
+async function firestoreHelper(action, { key, updates, config, db }) {
+    const collections = { price: config.priceCollection || 'asset_prices', status: config.computationStatusCollection || 'computation_status', };
+    switch (action) {
+        case 'checkAvailability': try { const snapshot = await db.collection(collections.price).limit(1).get(); return !snapshot.empty; } catch (e) { return false; }
+        case 'fetchStatus': { if (!key) throw new Error('fetchStatus requires a key'); const docRef = db.collection(collections.status).doc(key); const snap = await docRef.get(); return snap.exists ? snap.data() : {}; }
+        case 'updateStatus': { if (!key) throw new Error('updateStatus requires a key'); if (!updates || Object.keys(updates).length === 0) return; const docRef = db.collection(collections.status).doc(key); await docRef.set(updates, { merge: true }); return true; }
+        default: throw new Error(`Unknown action: ${action}`);
     }
 }
 
-async function fetchComputationStatus(dateStr, config, { db }) {
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc(dateStr);
-    const snap = await docRef.get();
-    return snap.exists ? snap.data() : {};
-}
-
-async function fetchGlobalComputationStatus(config, { db }) {
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc('global_status');
-    const snap       = await docRef.get();
-    return snap.exists ? snap.data() : {};
-}
-
-async function updateComputationStatus(dateStr, updates, config, { db }) {
-    if (!updates || Object.keys(updates).length === 0) return;
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc(dateStr);
-    await docRef.set(updates, { merge: true });
-}
-
-async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
-    if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc('global_status');
-    const flattenUpdates = {};
-    for (const [date, statuses] of Object.entries(updatesByDate)) {
-        for (const [calc, status] of Object.entries(statuses)) {
-            flattenUpdates[`${date}.${calc}`] = status;
-        }
-    }
-    try {
-        await docRef.update(flattenUpdates);
-    } catch (err) {
-        if (err.code === 5) {
-            const deepObj = {};
-            for (const [date, statuses] of Object.entries(updatesByDate)) {
-                deepObj[date] = statuses;
-            }
-            await docRef.set(deepObj, { merge: true });
-        } else {
-            throw err;
-        }
-    }
-}
+async function checkPriceDataAvailability(config, dependencies) { return firestoreHelper('checkAvailability', { config, db: dependencies.db }); }
+async function fetchComputationStatus(dateStr, config, { db }) { return firestoreHelper('fetchStatus', { key: dateStr, config, db }); }
+async function fetchGlobalComputationStatus(config, { db }) { return firestoreHelper('fetchStatus', { key: 'global_status', config, db }); }
+async function updateComputationStatus(dateStr, updates, config, { db }) { return firestoreHelper('updateStatus', { key: dateStr, updates, config, db }); }
 
+/**
+ * --- REFACTORED: fetchExistingResults ---
+ * Transparently handles both standard documents and auto-sharded documents.
+ * 1. Fetches the doc.
+ * 2. Checks for `_sharded: true` flag.
+ * 3. If sharded, fetches subcollection and merges data back into a single object.
+ */
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
-    for (const calc of calcsInPass) {
-        if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
-        if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
-    }
+    for (const calc of calcsInPass) { if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); if (includeSelf && calc.isHistorical) calcsToFetch.add(normalizeName(calc.name)); }
     if (!calcsToFetch.size) return {};
     const fetched = {};
     const docRefs = [];
     const names   = [];
+
+    // 1. Prepare Reads
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) {
-            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection).doc(name));
-            names.push(name);
-        }
+        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(m.category || 'unknown').collection(config.computationsSubcollection).doc(name)); names.push(name); }
     }
+
     if (docRefs.length) {
         const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } });
+        const hydrationPromises = [];
+
+        // 2. Process Initial Snapshots
+        snaps.forEach((doc, i) => { const name = names[i]; if (!doc.exists) return; const data = doc.data(); if (data._sharded === true) { hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); } else if (data._completed) { fetched[name] = data; } }); // CHECK FOR AUTO-SHARDING FLAG
+        
+
+        // 3. Hydrate Sharded Data in Parallel
+        if (hydrationPromises.length > 0) { const hydratedResults = await Promise.all(hydrationPromises); hydratedResults.forEach(res => { fetched[res.name] = res.data; }); }
     }
     return fetched;
 }
 
+/**
+ * Helper: Fetches all docs in the '_shards' subcollection and merges them.
+ */
+async function hydrateAutoShardedResult(docRef, resultName) {
+    // Determine subcollection name (defaulting to '_shards')
+    const shardsCol = docRef.collection('_shards');
+    const snapshot = await shardsCol.get();
+    
+    const assembledData = { _completed: true }; // Rebuild the object
+    
+    snapshot.forEach(doc => { const chunk = doc.data(); Object.assign(assembledData, chunk); });
+    
+    // Remove internal flags if they leaked into the shards
+    delete assembledData._sharded;
+    delete assembledData._completed;
+
+    return { name: resultName, data: assembledData };
+}
+
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
-    const controller     = new ComputationController(config, deps);
-    const calcs          = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c =>
-        c.manifest.rootDataDependencies.includes('portfolio') ||
-        c.manifest.rootDataDependencies.includes('history')
-    );
-
+    const controller = new ComputationController(config, deps);
+    const calcs = Object.values(state).filter(c => c && c.manifest);
+    const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
     if (streamingCalcs.length === 0) return;
-
+    
     logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
-
     await controller.loader.loadMappings();
-    const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
-
-    const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+    const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-    const needsTradingHistory     = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter                 = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
-
-    let yP_chunk = {};
-    let tH_chunk = {};
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
+    let yP_chunk = {}, tH_chunk = {};
     for await (const tP_chunk of tP_iter) {
         if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
         if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-
-        const promises = streamingCalcs.map(calc =>
-            controller.executor.executePerUser(
-                calc,
-                calc.manifest,
-                dateStr,
-                tP_chunk,
-                yP_chunk,
-                tH_chunk,
-                fetchedDeps,
-                previousFetchedDeps
-            )
-        );
+        const promises = streamingCalcs.map(calc => controller.executor.executePerUser(calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, fetchedDeps, previousFetchedDeps));
         await Promise.all(promises);
     }
     logger.log('INFO', `[${passName}] Streaming complete.`);
@@ -255,18 +179,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         const prevStr = prev.toISOString().slice(0, 10);
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
-
     const state = {};
-    for (const c of calcs) {
-        try {
-            const inst = new c.class();
-            inst.manifest = c;
-            state[normalizeName(c.name)] = inst;
-            logger.log('INFO', `${c.name} calculation running for ${dStr}`);
-        }
-        catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
-    }
-
+    for (const c of calcs) { try { const inst = new c.class(); inst.manifest = c; state[normalizeName(c.name)] = inst; logger.log('INFO', `${c.name} calculation running for ${dStr}`); } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } }
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
@@ -275,12 +189,10 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     const controller = new ComputationController(config, deps);
     const dStr = date.toISOString().slice(0, 10);
     const state = {};
-
     for (const mCalc of calcs) {
         try {
             deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
-            const inst = new mCalc.class();
-            inst.manifest = mCalc;
+            const inst = new mCalc.class(); inst.manifest = mCalc;
             await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
             state[normalizeName(mCalc.name)] = inst;
         } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
@@ -290,101 +202,46 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 
 /**
  * --- REFACTORED: commitResults ---
- * Commits results individually per calculation.
- * If one calculation fails (e.g. size limit), others still succeed.
- * UPDATED: Writes the HASH to the status document.
+ * Automatically detects result size.
+ * If > 900KB, it splits the result into chunks and writes to a subcollection.
+ * If < 900KB, it writes normally.
  */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const successUpdates = {};
     const schemas = [];
 
-    // Iterate PER CALCULATION to isolate failures
     for (const name in stateObj) {
         const calc = stateObj[name];
         let hasData = false;
 
         try {
             const result = await calc.getResult();
-            if (!result) {
-                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`);
-                continue;
-            }
-
-            const standardRes   = {};
-            const shardedWrites = [];
-            const calcWrites    = []; 
-
-            // 1. Separate Standard and Sharded Data
-            for (const key in result) {
-                if (key.startsWith('sharded_')) {
-                    const sData = result[key];
-                    for (const colName in sData) {
-                        const docsMap = sData[colName];
-                        for (const docId in docsMap) {
-                            const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId);
-                            shardedWrites.push({
-                                ref,
-                                data: { ...docsMap[docId], _completed: true }
-                            });
-                        }
-                    }
-                    if (Object.keys(sData).length > 0) hasData = true;
-                } else {
-                    standardRes[key] = result[key];
-                }
-            }
-
-            // 2. Prepare Standard Result Write
-            if (Object.keys(standardRes).length) {
-                validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-                standardRes._completed = true;
-
-                const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                    .collection(config.computationsSubcollection).doc(name);
+            if (!result) { deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`); continue; }
 
-                calcWrites.push({
-                    ref: docRef,
-                    data: standardRes
-                });
-                hasData = true;
-            }
+            const mainDocRef = deps.db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
 
-            // 3. Queue Schema (Safe to accumulate)
-            if (calc.manifest.class.getSchema) {
-                const { class: _cls, ...safeMetadata } = calc.manifest;
-                schemas.push({
-                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata
-                });
+            // AUTO-SHARDING LOGIC
+            const updates = await prepareAutoShardedWrites(result, mainDocRef, deps.logger);
+            
+            // Collect Schemas if present
+            if (calc.manifest.class.getSchema) { 
+                const { class: _cls, ...safeMetadata } = calc.manifest; 
+                schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata }); 
             }
 
-            // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
-            if (hasData) {
-                const allWritesForCalc = [...calcWrites, ...shardedWrites];
-
-                if (allWritesForCalc.length > 0) {
-                    await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
-
-                    // --- CRITICAL UPDATE: Store the Smart Hash ---
-                    successUpdates[name] = calc.manifest.hash || true;
-                    
-                    deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
-                } else {
-                    deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`);
-                }
-            } else {
-                deps.logger.log('INFO', `${name} for ${dStr}: - Empty`);
+            if (updates.length > 0) {
+                await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                successUpdates[name] = calc.manifest.hash || true;
+                const isSharded = updates.some(u => u.data._sharded === true);
+                deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written ${isSharded ? 'Sharded' : 'Standard'})`);
+            } else { 
+                deps.logger.log('INFO', `${name} for ${dStr}: - Empty Data`); 
             }
 
-        } catch (e) {
-            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
-        }
+        } catch (e) { deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`); }
     }
 
-    // Save Schemas (Best effort, isolated)
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
-
-    // Update Status Document (Only for the ones that succeeded)
     if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
         deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
@@ -393,151 +250,151 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 }
 
 /**
- * --- UPDATED: runBatchPriceComputation ---
+ * Accurately calculates the size of a value according to Firestore storage rules.
+ * Reference: https://firebase.google.com/docs/firestore/storage-size
  */
+function calculateFirestoreBytes(value) {
+    if (value === null) return 1;
+    if (value === undefined) return 0; // Firestore drops undefined fields
+    if (typeof value === 'boolean') return 1;
+    if (typeof value === 'number') return 8; // All numbers are 64-bit doubles or integers
+    if (typeof value === 'string') return Buffer.byteLength(value, 'utf8') + 1;
+    if (value instanceof Date) return 8; // Timestamps are 8 bytes
+    
+    // Handle References (approximate based on path length)
+    if (value.constructor && value.constructor.name === 'DocumentReference') {
+        // Path string + 16 bytes for the reference type overhead
+        return Buffer.byteLength(value.path, 'utf8') + 16;
+    }
+    
+    // Handle Arrays: Sum of all values
+    if (Array.isArray(value)) {
+        let sum = 0;
+        for (const item of value) sum += calculateFirestoreBytes(item);
+        return sum;
+    }
+    
+    // Handle Objects (Maps): Sum of (Key + 1 + Value)
+    if (typeof value === 'object') {
+        let sum = 0;
+        for (const k in value) {
+            if (Object.prototype.hasOwnProperty.call(value, k)) {
+                // Key size (utf8 + 1) + Value size
+                sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]);
+            }
+        }
+        return sum;
+    }
+
+    return 0; // Fallback
+}
+
+
+async function prepareAutoShardedWrites(result, docRef, logger) {
+    const SAFETY_THRESHOLD_BYTES = 1000 * 1024; // 1MB Limit (We target just under this)
+    const OVERHEAD_ALLOWANCE = 20 * 1024; // 20KB Safety margin for document path & metadata
+    const CHUNK_LIMIT = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
+    const totalSize = calculateFirestoreBytes(result); // 1. Calculate Total Size Once (O(N))
+    const docPathSize = Buffer.byteLength(docRef.path, 'utf8') + 16; // Add the size of the document path itself (Firestore counts this against the 1MB limit)
+
+    if ((totalSize + docPathSize) < CHUNK_LIMIT) { const data = { ...result, _completed: true, _sharded: false }; return [{ ref: docRef, data, options: { merge: true } }]; } // CASE A: Fits in one document
+
+    logger.log('INFO', `[AutoShard] Result size ~${Math.round(totalSize/1024)}KB exceeds limit. Sharding...`);
+
+    const writes = [];
+    const shardCollection = docRef.collection('_shards');
+    
+    let currentChunk = {};
+    let currentChunkSize = 0;
+    let shardIndex = 0;
+
+    
+    for (const [key, value] of Object.entries(result)) { // 2. Efficient O(N) Loop
+        if (key.startsWith('_')) continue;
+        const keySize = Buffer.byteLength(key, 'utf8') + 1;         // Calculate size of just this item
+        const valueSize = calculateFirestoreBytes(value);
+        const itemSize = keySize + valueSize;
+
+        if (currentChunkSize + itemSize > CHUNK_LIMIT) { // Check if adding this item would overflow the current chunk
+            // Flush current chunk
+            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); // Overwrite
+            shardIndex++;
+            currentChunk = {};
+            currentChunkSize = 0;
+        }
+
+        // Add to current chunk
+        currentChunk[key] = value;
+        currentChunkSize += itemSize;
+    }
+
+    // Flush final chunk
+    if (Object.keys(currentChunk).length > 0) { writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); }
+
+    // Pointer Document
+    const pointerData = { _completed: true,  _sharded: true, _shardCount: shardIndex + 1, _lastUpdated: new Date().toISOString() };
+
+    // Use merge: false to ensure we overwrite any previous non-sharded blob
+    writes.push({ ref: docRef, data: pointerData, options: { merge: false } });
+
+    return writes;
+}
+
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
     const { logger, db, calculationUtils } = deps;
     const controller = new ComputationController(config, deps);
-
     const mappings = await controller.loader.loadMappings();
-
     let targetInstrumentIds = [];
     if (targetTickers && targetTickers.length > 0) {
         const tickerToInst = mappings.tickerToInstrument || {};
         targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        if (targetInstrumentIds.length === 0) {
-            logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
-            return;
-        }
+        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } 
     }
-
     const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-
-    if (!allShardRefs.length) {
-        logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
-        return;
-    }
-
-    const OUTER_CONCURRENCY_LIMIT = 2;
-    const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50;
-
+    if (!allShardRefs.length) { logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.'); return; }
+    const OUTER_CONCURRENCY_LIMIT = 2, SHARD_BATCH_SIZE = 20, WRITE_BATCH_LIMIT = 50;
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
-
-    const shardChunks = [];
-    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
-        shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
-    }
-
+    const shardChunks = []; for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
     const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
-
     const chunkPromises = [];
     for (let index = 0; index < shardChunks.length; index++) {
         const shardChunkRefs = shardChunks[index];
         chunkPromises.push(outerLimit(async () => {
             try {
                 logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-
                 const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-
-                if (targetInstrumentIds.length > 0) {
-                    const requestedSet = new Set(targetInstrumentIds);
-                    for (const loadedInstrumentId in pricesData) {
-                        if (!requestedSet.has(loadedInstrumentId)) {
-                            delete pricesData[loadedInstrumentId];
-                        }
-                    }
-                }
-
+                if (targetInstrumentIds.length > 0) { const requestedSet = new Set(targetInstrumentIds); for (const loadedInstrumentId in pricesData) { if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } } }
                 const writes = [];
-
                 for (const dateStr of dateStrings) {
-                    const context = {
-                        mappings,
-                        prices: { history: pricesData },
-                        date: { today: dateStr },
-                        math: {
-                            extract: DataExtractor,
-                            history: HistoryExtractor,
-                            compute: MathPrimitives,
-                            aggregate: Aggregators,
-                            validate: Validators,
-                            signals: SignalPrimitives,
-                            schemas: SCHEMAS,
-                            distribution: DistributionAnalytics,
-                            TimeSeries: TimeSeries,
-                            priceExtractor: priceExtractor
-                        }
-                    };
-
+                    const dynamicMathContext = {};
+                    for (const [key, value] of Object.entries(mathLayer)) { dynamicMathContext[key] = value; if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} }
+                    const context = { mappings, prices: { history: pricesData }, date: { today: dateStr }, math: dynamicMathContext };
                     for (const calcManifest of calcs) {
                         try {
-                            const instance = new calcManifest.class();
-                            await instance.process(context);
-                            const result = await instance.getResult();
-
+                            const instance = new calcManifest.class(); await instance.process(context); const result = await instance.getResult();
                             if (result && Object.keys(result).length > 0) {
-                                let dataToWrite = result;
-                                if (result.by_instrument) dataToWrite = result.by_instrument;
-
+                                let dataToWrite = result; if (result.by_instrument) dataToWrite = result.by_instrument;
                                 if (Object.keys(dataToWrite).length > 0) {
-                                    const docRef = db.collection(config.resultsCollection).doc(dateStr)
-                                        .collection(config.resultsSubcollection).doc(calcManifest.category)
-                                        .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-
-                                    writes.push({
-                                        ref: docRef,
-                                        data: { ...dataToWrite, _completed: true },
-                                        options: { merge: true }
-                                    });
+                                    const docRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category).collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+                                    writes.push({ ref: docRef, data: { ...dataToWrite, _completed: true }, options: { merge: true } });
                                 }
                             }
-                        } catch (err) {
-                            logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
-                        }
+                        } catch (err) { logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`); }
                     }
                 }
-
                 if (writes.length > 0) {
-                    const commitBatches = [];
-                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
-                        commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
-                    }
-
+                    const commitBatches = []; for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
                     const commitLimit = pLimit(10);
-
                     await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                        const batch = db.batch();
-                        batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-
-                        try {
-                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
-                        } catch (commitErr) {
-                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
-                        }
+                        const batch = db.batch(); batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+                        try { await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`); } catch (commitErr) { logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); }
                     })));
                 }
-
-            } catch (chunkErr) {
-                logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
-            }
+            } catch (chunkErr) { logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message }); }
         }));
     }
-
     await Promise.all(chunkPromises);
     logger.log('INFO', '[BatchPrice] Optimization pass complete.');
 }
 
-module.exports = {
-    groupByPass,
-    checkRootDependencies,
-    checkRootDataAvailability,
-    fetchExistingResults,
-    fetchComputationStatus,
-    fetchGlobalComputationStatus,
-    updateComputationStatus,
-    updateGlobalComputationStatus,
-    runStandardComputationPass,
-    runMetaComputationPass,
-    runBatchPriceComputation
-};
+module.exports = { groupByPass, checkRootDependencies, checkRootDataAvailability, fetchExistingResults, fetchComputationStatus, fetchGlobalComputationStatus, updateComputationStatus, runStandardComputationPass, runMetaComputationPass, runBatchPriceComputation };