bulltrackers-module 1.0.215 → 1.0.216

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,8 +1,13 @@
 /**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: 'commitResults' now records the CODE HASH in the status document
- * instead of a boolean, enabling auto-invalidation on code changes.
- * UPDATED: Dynamic math context loading from 'layers/index.js' to support modular layers.
+ * FILENAME: computation-system/helpers/orchestration_helpers.js
+ * FEATURE: Dynamic Auto-Sharding (Transparent 1MB Limit Handling)
+ * * DESCRIPTION:
+ * This module orchestrates the execution of computations. It handles:
+ * 1. Data Availability Checks
+ * 2. Dependency Injection (fetching results from previous passes)
+ * 3. Transparent Auto-Sharding:
+ * - Writes: Automatically detects if a result > 900KB. Splits it into a '_shards' subcollection.
+ * - Reads: Automatically detects sharded pointers and re-assembles the data.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -13,60 +18,27 @@ const {
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
-
-// --- DYNAMIC LAYER LOADING ---
-// Replaces the old static import from 'math_primitives.js'
 const mathLayer = require('../layers/index.js');
-
 const pLimit = require('p-limit');
 
-// Mappings to ensure backward compatibility with existing calculation code
-// (e.g. allowing 'math.compute' to resolve to 'MathPrimitives')
+// Mappings for backward compatibility
 const LEGACY_MAPPING = {
-    DataExtractor:         'extract',
-    HistoryExtractor:      'history',
-    MathPrimitives:        'compute',
-    Aggregators:           'aggregate',
-    Validators:            'validate',
-    SignalPrimitives:      'signals',
-    SCHEMAS:               'schemas',
-    DistributionAnalytics: 'distribution',
-    TimeSeries:            'TimeSeries',
-    priceExtractor:        'priceExtractor',
-    InsightsExtractor:     'insights',
-    UserClassifier:        'classifier',
-    CognitiveBiases:       'bias',
-    SkillAttribution:      'skill',
-    Psychometrics:         'psychometrics'
+    DataExtractor: 'extract', HistoryExtractor: 'history', MathPrimitives: 'compute', Aggregators: 'aggregate', Validators: 'validate', SignalPrimitives: 'signals', SCHEMAS: 'schemas', DistributionAnalytics: 'distribution', TimeSeries: 'TimeSeries', priceExtractor: 'priceExtractor', InsightsExtractor: 'insights', UserClassifier: 'classifier', CognitiveBiases: 'bias', SkillAttribution: 'skill', Psychometrics: 'psychometrics'
 };
 
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
-/**
- * --- PASSIVE DATA VALIDATION ---
- */
 function validateResultPatterns(logger, calcName, results, category) {
     if (category === 'speculator' || category === 'speculators') return;
-    const tickers    = Object.keys(results);
-    const totalItems = tickers.length;
-    if (totalItems < 5) return;
-    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
-    if (!sampleTicker) return;
-    const keys = Object.keys(results[sampleTicker]);
-    keys.forEach(key => {
+    const tickers = Object.keys(results); const totalItems = tickers.length; if (totalItems < 5) return;
+    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object'); if (!sampleTicker) return;
+    Object.keys(results[sampleTicker]).forEach(key => {
         if (key.startsWith('_')) return;
-        let nullCount      = 0;
-        let nanCount       = 0;
-        let undefinedCount = 0;
-        for (const t of tickers) {
-            const val = results[t][key];
-            if (val === null) nullCount++;
-            if (val === undefined) undefinedCount++;
-            if (typeof val === 'number' && isNaN(val)) nanCount++;
-        }
-        if (nanCount === totalItems) { logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
-        } else if (undefinedCount === totalItems) { logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`); }
-        else if (nullCount > (totalItems * 0.9)) { logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`); }
+        let nullCount = 0, nanCount = 0, undefinedCount = 0;
+        for (const t of tickers) { const val = results[t][key]; if (val === null) nullCount++; if (val === undefined) undefinedCount++; if (typeof val === 'number' && isNaN(val)) nanCount++; }
+        if (nanCount === totalItems) logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of items.`);
+        else if (undefinedCount === totalItems) logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of items.`);
+        else if (nullCount > (totalItems * 0.9)) logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
     });
 }
 
@@ -74,11 +46,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
-        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
-        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
-        else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
+        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+        else if (dep === 'price' && !rootDataStatus.hasPrices) missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -88,115 +60,111 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     let portfolioRefs = [], historyRefs = [];
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false, insightsData = null, socialData = null;
-
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs        (config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio  = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights           (config, dependencies, dateStr).then(r => { insightsData  = r; hasInsights   = !!r; }));
-        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
-        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
-
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         if (dateToProcess >= earliestDates.price) { tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; })); }
         await Promise.all(tasks);
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-
-        return {
-            portfolioRefs,
-            historyRefs,
-            todayInsights: insightsData,
-            todaySocialPostInsights: socialData,
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices },
-            yesterdayPortfolioRefs: null
-        };
-
+        return { portfolioRefs, historyRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }, yesterdayPortfolioRefs: null };
     } catch (err) { logger.log('ERROR', `Error checking data: ${err.message}`); return null; }
 }
 
 async function firestoreHelper(action, { key, updates, config, db }) {
-  const collections = { price: config.priceCollection || 'asset_prices', status: config.computationStatusCollection || 'computation_status', };
-
-  switch (action) {
-    case 'checkAvailability': {
-      try { const snapshot = await db.collection(collections.price).limit(1).get(); return !snapshot.empty; } catch (e) { return false; } }
-
-    case 'fetchStatus': {
-      if (!key) throw new Error('fetchStatus requires a key');
-      const docRef = db.collection(collections.status).doc(key);
-      const snap = await docRef.get();
-      return snap.exists ? snap.data() : {};
+    const collections = { price: config.priceCollection || 'asset_prices', status: config.computationStatusCollection || 'computation_status', };
+    switch (action) {
+        case 'checkAvailability': try { const snapshot = await db.collection(collections.price).limit(1).get(); return !snapshot.empty; } catch (e) { return false; }
+        case 'fetchStatus': { if (!key) throw new Error('fetchStatus requires a key'); const docRef = db.collection(collections.status).doc(key); const snap = await docRef.get(); return snap.exists ? snap.data() : {}; }
+        case 'updateStatus': { if (!key) throw new Error('updateStatus requires a key'); if (!updates || Object.keys(updates).length === 0) return; const docRef = db.collection(collections.status).doc(key); await docRef.set(updates, { merge: true }); return true; }
+        default: throw new Error(`Unknown action: ${action}`);
     }
-
-    case 'updateStatus': {
-      if (!key) throw new Error('updateStatus requires a key');
-      if (!updates || Object.keys(updates).length === 0) return;
-      const docRef = db.collection(collections.status).doc(key);
-      await docRef.set(updates, { merge: true });
-      return true;
-    }
-
-    default: throw new Error(`Unknown action: ${action}`); }
 }
 
-async function checkPriceDataAvailability    (config, dependencies)             { return firestoreHelper('checkAvailability', { config, db: dependencies.db }); }
-async function fetchComputationStatus        (dateStr, config, { db })          { return firestoreHelper('fetchStatus', { key: dateStr, config, db }); }
-async function fetchGlobalComputationStatus  (config, { db })                   { return firestoreHelper('fetchStatus', { key: 'global_status', config, db }); }
-async function updateComputationStatus       (dateStr, updates, config, { db }) { return firestoreHelper('updateStatus', { key: dateStr, updates, config, db }); }
-
+async function checkPriceDataAvailability(config, dependencies) { return firestoreHelper('checkAvailability', { config, db: dependencies.db }); }
+async function fetchComputationStatus(dateStr, config, { db }) { return firestoreHelper('fetchStatus', { key: dateStr, config, db }); }
+async function fetchGlobalComputationStatus(config, { db }) { return firestoreHelper('fetchStatus', { key: 'global_status', config, db }); }
+async function updateComputationStatus(dateStr, updates, config, { db }) { return firestoreHelper('updateStatus', { key: dateStr, updates, config, db }); }
 
+/**
+ * --- REFACTORED: fetchExistingResults ---
+ * Transparently handles both standard documents and auto-sharded documents.
+ * 1. Fetches the doc.
+ * 2. Checks for `_sharded: true` flag.
+ * 3. If sharded, fetches subcollection and merges data back into a single object.
+ */
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
-    for (const calc of calcsInPass) { if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); } if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); } }
+    for (const calc of calcsInPass) { if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); if (includeSelf && calc.isHistorical) calcsToFetch.add(normalizeName(calc.name)); }
     if (!calcsToFetch.size) return {};
     const fetched = {};
     const docRefs = [];
     const names   = [];
+
+    // 1. Prepare Reads
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr) .collection(config.resultsSubcollection).doc(m.category || 'unknown') .collection(config.computationsSubcollection).doc(name)); names.push(name); } }
-    if (docRefs.length) { const snaps = await db.getAll(...docRefs); snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } }); }
+        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(m.category || 'unknown').collection(config.computationsSubcollection).doc(name)); names.push(name); }
+    }
+
+    if (docRefs.length) {
+        const snaps = await db.getAll(...docRefs);
+        const hydrationPromises = [];
+
+        // 2. Process Initial Snapshots
+        snaps.forEach((doc, i) => { const name = names[i]; if (!doc.exists) return; const data = doc.data(); if (data._sharded === true) { hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); } else if (data._completed) { fetched[name] = data; } }); // CHECK FOR AUTO-SHARDING FLAG
+        
+
+        // 3. Hydrate Sharded Data in Parallel
+        if (hydrationPromises.length > 0) { const hydratedResults = await Promise.all(hydrationPromises); hydratedResults.forEach(res => { fetched[res.name] = res.data; }); }
+    }
     return fetched;
 }
 
+/**
+ * Helper: Fetches all docs in the '_shards' subcollection and merges them.
+ */
+async function hydrateAutoShardedResult(docRef, resultName) {
+    // Determine subcollection name (defaulting to '_shards')
+    const shardsCol = docRef.collection('_shards');
+    const snapshot = await shardsCol.get();
+    
+    const assembledData = { _completed: true }; // Rebuild the object
+    
+    snapshot.forEach(doc => { const chunk = doc.data(); Object.assign(assembledData, chunk); });
+    
+    // Remove internal flags if they leaked into the shards
+    delete assembledData._sharded;
+    delete assembledData._completed;
+
+    return { name: resultName, data: assembledData };
+}
+
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
-    const controller     = new ComputationController(config, deps);
-    const calcs          = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history') );
-
+    const controller = new ComputationController(config, deps);
+    const calcs = Object.values(state).filter(c => c && c.manifest);
+    const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
     if (streamingCalcs.length === 0) return;
-
+    
     logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
-
     await controller.loader.loadMappings();
-    const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
-
-    const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+    const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-    const needsTradingHistory     = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter                 = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
-
-    let yP_chunk = {};
-    let tH_chunk = {};
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
+    let yP_chunk = {}, tH_chunk = {};
     for await (const tP_chunk of tP_iter) {
         if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
         if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-
-        const promises = streamingCalcs.map(calc =>
-            controller.executor.executePerUser(
-                calc,
-                calc.manifest,
-                dateStr,
-                tP_chunk,
-                yP_chunk,
-                tH_chunk,
-                fetchedDeps,
-                previousFetchedDeps
-            )
-        );
+        const promises = streamingCalcs.map(calc => controller.executor.executePerUser(calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, fetchedDeps, previousFetchedDeps));
         await Promise.all(promises);
     }
     logger.log('INFO', `[${passName}] Streaming complete.`);
@@ -211,13 +179,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         const prevStr = prev.toISOString().slice(0, 10);
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
-
     const state = {};
-    for (const c of calcs) {
-        try { const inst = new c.class(); inst.manifest = c; state[normalizeName(c.name)] = inst; logger.log('INFO', `${c.name} calculation running for ${dStr}`); }
-        catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
-    }
-
+    for (const c of calcs) { try { const inst = new c.class(); inst.manifest = c; state[normalizeName(c.name)] = inst; logger.log('INFO', `${c.name} calculation running for ${dStr}`); } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } }
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
     return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
@@ -226,12 +189,10 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     const controller = new ComputationController(config, deps);
     const dStr = date.toISOString().slice(0, 10);
     const state = {};
-
     for (const mCalc of calcs) {
         try {
             deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
-            const inst = new mCalc.class();
-            inst.manifest = mCalc;
+            const inst = new mCalc.class(); inst.manifest = mCalc;
             await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
             state[normalizeName(mCalc.name)] = inst;
         } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
@@ -241,15 +202,14 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 
 /**
  * --- REFACTORED: commitResults ---
- * Commits results individually per calculation.
- * If one calculation fails (e.g. size limit), others still succeed.
- * UPDATED: Writes the HASH to the status document.
+ * Automatically detects result size.
+ * If > 900KB, it splits the result into chunks and writes to a subcollection.
+ * If < 900KB, it writes normally.
  */
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const successUpdates = {};
     const schemas = [];
 
-    // Iterate PER CALCULATION to isolate failures
     for (const name in stateObj) {
         const calc = stateObj[name];
         let hasData = false;
@@ -258,49 +218,30 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const result = await calc.getResult();
             if (!result) { deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`); continue; }
 
-            const standardRes   = {};
-            const shardedWrites = [];
-            const calcWrites    = []; 
-
-            // 1. Separate Standard and Sharded Data
-            for (const key in result) {
-                if (key.startsWith('sharded_')) {
-                    const sData = result[key];
-                    for (const colName in sData) {
-                        const docsMap = sData[colName];
-                        for (const docId in docsMap) { const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId); shardedWrites.push({ ref, data: { ...docsMap[docId], _completed: true } }); } }
-                    if (Object.keys(sData).length > 0) hasData = true;
-                } else { standardRes[key] = result[key]; }
+            const mainDocRef = deps.db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
+
+            // AUTO-SHARDING LOGIC
+            const updates = await prepareAutoShardedWrites(result, mainDocRef, deps.logger);
+            
+            // Collect Schemas if present
+            if (calc.manifest.class.getSchema) { 
+                const { class: _cls, ...safeMetadata } = calc.manifest; 
+                schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata }); 
             }
 
-            // 2. Prepare Standard Result Write
-            if (Object.keys(standardRes).length) {
-                validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-                standardRes._completed = true;
-                const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(calc.manifest.category) .collection(config.computationsSubcollection).doc(name);
-                calcWrites.push({ ref: docRef, data: standardRes });
-                hasData = true;
+            if (updates.length > 0) {
+                await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                successUpdates[name] = calc.manifest.hash || true;
+                const isSharded = updates.some(u => u.data._sharded === true);
+                deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written ${isSharded ? 'Sharded' : 'Standard'})`);
+            } else { 
+                deps.logger.log('INFO', `${name} for ${dStr}: - Empty Data`); 
             }
 
-            // 3. Queue Schema (Safe to accumulate)
-            if (calc.manifest.class.getSchema) { const { class: _cls, ...safeMetadata } = calc.manifest; schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata }); }
-
-            // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
-            if (hasData) {
-                const allWritesForCalc = [...calcWrites, ...shardedWrites];
-                if (allWritesForCalc.length > 0) {
-                    await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
-                    successUpdates[name] = calc.manifest.hash || true;
-                    deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
-                } else { deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`); }
-            } else { deps.logger.log('INFO', `${name} for ${dStr}: - Empty`); }
         } catch (e) { deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`); }
     }
 
-    // Save Schemas (Best effort, isolated)
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
-
-    // Update Status Document (Only for the ones that succeeded)
     if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
         deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
@@ -309,116 +250,151 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 }
 
 /**
- * --- UPDATED: runBatchPriceComputation ---
+ * Accurately calculates the size of a value according to Firestore storage rules.
+ * Reference: https://firebase.google.com/docs/firestore/storage-size
  */
+function calculateFirestoreBytes(value) {
+    if (value === null) return 1;
+    if (value === undefined) return 0; // Firestore drops undefined fields
+    if (typeof value === 'boolean') return 1;
+    if (typeof value === 'number') return 8; // All numbers are 64-bit doubles or integers
+    if (typeof value === 'string') return Buffer.byteLength(value, 'utf8') + 1;
+    if (value instanceof Date) return 8; // Timestamps are 8 bytes
+    
+    // Handle References (approximate based on path length)
+    if (value.constructor && value.constructor.name === 'DocumentReference') {
+        // Path string + 16 bytes for the reference type overhead
+        return Buffer.byteLength(value.path, 'utf8') + 16;
+    }
+    
+    // Handle Arrays: Sum of all values
+    if (Array.isArray(value)) {
+        let sum = 0;
+        for (const item of value) sum += calculateFirestoreBytes(item);
+        return sum;
+    }
+    
+    // Handle Objects (Maps): Sum of (Key + 1 + Value)
+    if (typeof value === 'object') {
+        let sum = 0;
+        for (const k in value) {
+            if (Object.prototype.hasOwnProperty.call(value, k)) {
+                // Key size (utf8 + 1) + Value size
+                sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]);
+            }
+        }
+        return sum;
+    }
+
+    return 0; // Fallback
+}
+
+
+async function prepareAutoShardedWrites(result, docRef, logger) {
+    const SAFETY_THRESHOLD_BYTES = 1000 * 1024; // 1MB Limit (We target just under this)
+    const OVERHEAD_ALLOWANCE = 20 * 1024; // 20KB Safety margin for document path & metadata
+    const CHUNK_LIMIT = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
+    const totalSize = calculateFirestoreBytes(result); // 1. Calculate Total Size Once (O(N))
+    const docPathSize = Buffer.byteLength(docRef.path, 'utf8') + 16; // Add the size of the document path itself (Firestore counts this against the 1MB limit)
+
+    if ((totalSize + docPathSize) < CHUNK_LIMIT) { const data = { ...result, _completed: true, _sharded: false }; return [{ ref: docRef, data, options: { merge: true } }]; } // CASE A: Fits in one document
+
+    logger.log('INFO', `[AutoShard] Result size ~${Math.round(totalSize/1024)}KB exceeds limit. Sharding...`);
+
+    const writes = [];
+    const shardCollection = docRef.collection('_shards');
+    
+    let currentChunk = {};
+    let currentChunkSize = 0;
+    let shardIndex = 0;
+
+    
+    for (const [key, value] of Object.entries(result)) { // 2. Efficient O(N) Loop
+        if (key.startsWith('_')) continue;
+        const keySize = Buffer.byteLength(key, 'utf8') + 1;         // Calculate size of just this item
+        const valueSize = calculateFirestoreBytes(value);
+        const itemSize = keySize + valueSize;
+
+        if (currentChunkSize + itemSize > CHUNK_LIMIT) { // Check if adding this item would overflow the current chunk
+            // Flush current chunk
+            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); // Overwrite
+            shardIndex++;
+            currentChunk = {};
+            currentChunkSize = 0;
+        }
+
+        // Add to current chunk
+        currentChunk[key] = value;
+        currentChunkSize += itemSize;
+    }
+
+    // Flush final chunk
+    if (Object.keys(currentChunk).length > 0) { writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); }
+
+    // Pointer Document
+    const pointerData = { _completed: true,  _sharded: true, _shardCount: shardIndex + 1, _lastUpdated: new Date().toISOString() };
+
+    // Use merge: false to ensure we overwrite any previous non-sharded blob
+    writes.push({ ref: docRef, data: pointerData, options: { merge: false } });
+
+    return writes;
+}
+
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
     const { logger, db, calculationUtils } = deps;
     const controller = new ComputationController(config, deps);
-
     const mappings = await controller.loader.loadMappings();
-
     let targetInstrumentIds = [];
     if (targetTickers && targetTickers.length > 0) {
         const tickerToInst = mappings.tickerToInstrument || {};
         targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } }
-
+        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } 
+    }
     const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-
     if (!allShardRefs.length) { logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.'); return; }
-
-    const OUTER_CONCURRENCY_LIMIT = 2;
-    const SHARD_BATCH_SIZE        = 20;
-    const WRITE_BATCH_LIMIT       = 50;
-
+    const OUTER_CONCURRENCY_LIMIT = 2, SHARD_BATCH_SIZE = 20, WRITE_BATCH_LIMIT = 50;
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
-
-    const shardChunks = [];
-    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
-
+    const shardChunks = []; for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
     const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
-
     const chunkPromises = [];
     for (let index = 0; index < shardChunks.length; index++) {
         const shardChunkRefs = shardChunks[index];
         chunkPromises.push(outerLimit(async () => {
             try {
                 logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-
                 const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-
-                if (targetInstrumentIds.length > 0) {
-                    const requestedSet = new Set(targetInstrumentIds);
-                    for (const loadedInstrumentId in pricesData) { if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } }
-                }
-
+                if (targetInstrumentIds.length > 0) { const requestedSet = new Set(targetInstrumentIds); for (const loadedInstrumentId in pricesData) { if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } } }
                 const writes = [];
-
                 for (const dateStr of dateStrings) {
-                    
-                    // --- DYNAMIC MATH CONTEXT CONSTRUCTION ---
                     const dynamicMathContext = {};
-                    for (const [key, value] of Object.entries(mathLayer)) { dynamicMathContext[key] = value;  if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} }
-
-                    const context = {
-                        mappings,
-                        prices: { history: pricesData },
-                        date: { today: dateStr },
-                        math: dynamicMathContext // Injected here
-                    };
-
+                    for (const [key, value] of Object.entries(mathLayer)) { dynamicMathContext[key] = value; if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} }
+                    const context = { mappings, prices: { history: pricesData }, date: { today: dateStr }, math: dynamicMathContext };
                     for (const calcManifest of calcs) {
                         try {
-                            const instance = new calcManifest.class();
-                            await instance.process(context);
-                            const result = await instance.getResult();
-
+                            const instance = new calcManifest.class(); await instance.process(context); const result = await instance.getResult();
                             if (result && Object.keys(result).length > 0) {
-                                let dataToWrite = result;
-                                if (result.by_instrument) dataToWrite = result.by_instrument;
-
+                                let dataToWrite = result; if (result.by_instrument) dataToWrite = result.by_instrument;
                                 if (Object.keys(dataToWrite).length > 0) {
-                                    const docRef = db.collection(config.resultsCollection).doc(dateStr) .collection(config.resultsSubcollection).doc(calcManifest.category) .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-
+                                    const docRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category).collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
                                     writes.push({ ref: docRef, data: { ...dataToWrite, _completed: true }, options: { merge: true } });
                                 }
                             }
                         } catch (err) { logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`); }
                     }
                 }
-
                 if (writes.length > 0) {
-                    const commitBatches = [];
-                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
-
+                    const commitBatches = []; for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
                     const commitLimit = pLimit(10);
-
                     await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                        const batch = db.batch();
-                        batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-
-                        try { await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
-                        } catch (commitErr) { logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); }
+                        const batch = db.batch(); batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
+                        try { await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`); } catch (commitErr) { logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); }
                     })));
                 }
-
             } catch (chunkErr) { logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message }); }
         }));
     }
-
     await Promise.all(chunkPromises);
     logger.log('INFO', '[BatchPrice] Optimization pass complete.');
 }
 
-module.exports = {
-    groupByPass,
-    checkRootDependencies,
-    checkRootDataAvailability,
-    fetchExistingResults,
-    fetchComputationStatus,
-    fetchGlobalComputationStatus,
-    updateComputationStatus,
-    runStandardComputationPass,
-    runMetaComputationPass,
-    runBatchPriceComputation
-};
+module.exports = { groupByPass, checkRootDependencies, checkRootDataAvailability, fetchExistingResults, fetchComputationStatus, fetchGlobalComputationStatus, updateComputationStatus, runStandardComputationPass, runMetaComputationPass, runBatchPriceComputation };

package/functions/computation-system/utils/utils.js

@@ -1,8 +1,5 @@
 /**
- * @fileoverview Computation system sub-pipes and utils.
- * REFACTORED: Now stateless and receive dependencies where needed.
- * FIXED: 'commitBatchInChunks' now respects Firestore 10MB size limit.
- * NEW: Added 'generateCodeHash' for version control.
+ * FILENAME: computation-system/utils/utils.js
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
@@ -12,27 +9,18 @@ const crypto = require('crypto');
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
 /**
- * Generates a SHA-256 hash of a code string, ignoring comments and whitespace.
- * This effectively versions the logic.
- * @param {string} codeString - The source code of the function/class.
- * @returns {string} The hex hash.
+ * Generates a SHA-256 hash of a code string.
  */
 function generateCodeHash(codeString) {
     if (!codeString) return 'unknown';
-    
-    // 1. Remove single-line comments (//...)
     let clean = codeString.replace(/\/\/.*$/gm, '');
-    // 2. Remove multi-line comments (/*...*/)
     clean = clean.replace(/\/\*[\s\S]*?\*\//g, '');
-    // 3. Remove all whitespace (spaces, tabs, newlines)
     clean = clean.replace(/\s+/g, '');
-    
     return crypto.createHash('sha256').update(clean).digest('hex');
 }
 
 /** * Stage 2: Commit a batch of writes in chunks 
- * FIXED: Now splits batches by SIZE (9MB limit) and COUNT (450 docs) 
- * to prevent "Request payload size exceeds the limit" errors.
+ * FIXED: Now respects write.options (e.g. { merge: false }) to allow overwrites/deletes.
  */
 async function commitBatchInChunks(config, deps, writes, operationName) {
     const { db, logger, calculationUtils } = deps;
@@ -43,17 +31,14 @@ async function commitBatchInChunks(config, deps, writes, operationName) {
         return;
     }
 
-    // Firestore Constraints
-    const MAX_BATCH_OPS = 300;          // Safety limit (Max 500)
-    const MAX_BATCH_BYTES = 9 * 1024 * 1024; // 9MB Safety limit (Max 10MB)
+    const MAX_BATCH_OPS = 300;
+    const MAX_BATCH_BYTES = 9 * 1024 * 1024; 
 
     let currentBatch = db.batch();
     let currentOpsCount = 0;
     let currentBytesEst = 0;
     let batchIndex = 1;
-    let totalChunks = 0; // We don't know total chunks in advance now due to dynamic sizing
 
-    // Helper to commit the current batch and reset
     const commitAndReset = async () => {
         if (currentOpsCount > 0) {
             try {
@@ -74,30 +59,25 @@ async function commitBatchInChunks(config, deps, writes, operationName) {
     };
 
     for (const write of writes) {
-        // 1. Estimate Size: JSON stringify is a decent proxy for Firestore payload size
-        // We handle potential circular refs or failures gracefully by assuming a minimum size
         let docSize = 100;
-        try {
-            if (write.data) docSize = JSON.stringify(write.data).length;
-        } catch (e) { /* ignore size check error */ }
+        try { if (write.data) docSize = JSON.stringify(write.data).length; } catch (e) { }
 
-        // 2. Warn if a SINGLE document is approaching the 1MB limit
         if (docSize > 900 * 1024) {
-            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB). This allows few ops per batch.`);
+            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB).`);
         }
 
-        // 3. Check if adding this write would overflow the batch
         if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) {
             await commitAndReset();
         }
 
-        // 4. Add to batch
-        currentBatch.set(write.ref, write.data, { merge: true });
+        // USE PROVIDED OPTIONS OR DEFAULT TO MERGE: TRUE
+        const options = write.options || { merge: true };
+        currentBatch.set(write.ref, write.data, options);
+        
         currentOpsCount++;
         currentBytesEst += docSize;
     }
 
-    // 5. Commit remaining
     await commitAndReset();
 }
 
@@ -112,10 +92,7 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/**
- * --- NEW HELPER ---
- * Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates.
- */
+/** Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates. */
 async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
@@ -149,22 +126,10 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
     return earliestDate;
 }
 
-/** * --- MODIFIED FUNCTION ---
- * Stage 5: Determine the earliest date from *all* source data.
- */
+/** Stage 5: Determine the earliest date from *all* source data. */
 async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
-    logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
-
-    const [
-        investorDate,
-        speculatorDate,
-        investorHistoryDate,
-        speculatorHistoryDate,
-        insightsDate,
-        socialDate,
-        priceDate
-    ] = await Promise.all([
+    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate, priceDate ] = await Promise.all([
         getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
@@ -185,17 +150,11 @@ async function getEarliestDataDates(config, deps) {
     const earliestInsightsDate = getMinDate(insightsDate);
     const earliestSocialDate = getMinDate(socialDate);
     const earliestPriceDate = getMinDate(priceDate);
-    const absoluteEarliest = getMinDate(
-        earliestPortfolioDate,
-        earliestHistoryDate,
-        earliestInsightsDate,
-        earliestSocialDate,
-        earliestPriceDate
-    );
+    const absoluteEarliest = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate, earliestPriceDate);
 
     const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
 
-    const result = {
+    return {
         portfolio: earliestPortfolioDate || new Date('2999-12-31T00:00:00Z'),
         history: earliestHistoryDate || new Date('2999-12-31T00:00:00Z'),
         insights: earliestInsightsDate || new Date('2999-12-31T00:00:00Z'),
@@ -203,71 +162,26 @@ async function getEarliestDataDates(config, deps) {
         price: earliestPriceDate || new Date('2999-12-31T00:00:00Z'),
         absoluteEarliest: absoluteEarliest || fallbackDate
     };
-
-    logger.log('INFO', 'Earliest data availability map built:', {
-        portfolio: result.portfolio.toISOString().slice(0, 10),
-        history: result.history.toISOString().slice(0, 10),
-        insights: result.insights.toISOString().slice(0, 10),
-        social: result.social.toISOString().slice(0, 10),
-        price: result.price.toISOString().slice(0, 10),
-        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
-    });
-
-    return result;
 }
 
-/**
- * NEW HELPER: Get the earliest date from price collection
- */
 async function getFirstDateFromPriceCollection(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     const collection = config.priceCollection || 'asset_prices';
-
     try {
-        logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
-
-        const snapshot = await withRetry(
-            () => db.collection(collection).limit(10).get(),
-            `GetPriceShards(${collection})`
-        );
-
-        if (snapshot.empty) {
-            logger.log('WARN', `No price shards found in ${collection}`);
-            return null;
-        }
-
+        const snapshot = await withRetry(() => db.collection(collection).limit(10).get(), `GetPriceShards(${collection})`);
         let earliestDate = null;
-
         snapshot.forEach(doc => {
             const shardData = doc.data();
             for (const instrumentId in shardData) {
                 const instrumentData = shardData[instrumentId];
                 if (!instrumentData.prices) continue;
-
-                const dates = Object.keys(instrumentData.prices)
-                    .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
-                    .sort();
-
-                if (dates.length > 0) {
-                    const firstDate = new Date(dates[0] + 'T00:00:00Z');
-                    if (!earliestDate || firstDate < earliestDate) {
-                        earliestDate = firstDate;
-                    }
-                }
+                const dates = Object.keys(instrumentData.prices).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort();
+                if (dates.length > 0) { const firstDate = new Date(dates[0] + 'T00:00:00Z'); if (!earliestDate || firstDate < earliestDate) earliestDate = firstDate; }
             }
         });
-
-        if (earliestDate) {
-            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
-        }
-
         return earliestDate;
-
-    } catch (e) {
-        logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message });
-        return null;
-    }
+    } catch (e) { logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message }); return null; }
 }
 
 module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates, generateCodeHash };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.215",
+  "version": "1.0.216",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [