bulltrackers-module 1.0.218 → 1.0.220

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,129 +0,0 @@
-/**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
- * FIXED: 'storedStatus.substring' crash and 'missing dependency' log clarity.
- */
-
-const {
-    groupByPass,
-    checkRootDataAvailability,
-    fetchExistingResults,
-    fetchComputationStatus,
-    updateComputationStatus,
-    runStandardComputationPass,
-    runMetaComputationPass,
-    checkRootDependencies,
-    runBatchPriceComputation
-} = require('./orchestration_helpers.js');
-
-const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
-
-const PARALLEL_BATCH_SIZE = 7;
-
-async function runComputationPass(config, dependencies, computationManifest) {
-    const { logger } = dependencies;
-    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
-    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
-
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
-
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
-    };
-
-    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
-
-    const passes          = groupByPass(computationManifest);
-    const calcsInThisPass = passes[passToRun] || [];
-
-    if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-
-    const passEarliestDate          = earliestDates.absoluteEarliest;
-    const endDateUTC                = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
-    const allExpectedDates          = getExpectedDateStrings(passEarliestDate, endDateUTC);
-    const priceBatchCalcs           = calcsInThisPass.filter(c => c.type === 'meta' && c.rootDataDependencies?.includes('price'));
-    const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
-
-    if (priceBatchCalcs.length > 0) {
-        try { await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); 
-        } catch (e) { logger.log('ERROR', 'Legacy Batch Price failed', e); }
-    }
-
-    if (standardAndOtherMetaCalcs.length === 0) return;
-
-    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
-        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        await Promise.all(batch.map(dateStr => runDateComputation(dateStr, passToRun, standardAndOtherMetaCalcs, config, dependencies, computationManifest)));
-    }
-}
-
-async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
-    const { logger }    = dependencies;
-    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    const dailyStatus   = await fetchComputationStatus(dateStr, config, dependencies);
-    const calcsToAttempt = [];
-    for (const calc of calcsInThisPass) {
-        const cName        = normalizeName(calc.name);
-        const storedStatus = dailyStatus[cName];
-        const currentHash  = calc.hash;
-
-        // 1. Dependency Check
-        if (calc.dependencies && calc.dependencies.length > 0) { const missing = calc.dependencies.filter(depName => !dailyStatus[normalizeName(depName)]); if (missing.length > 0) { logger.log('TRACE', `[Skip] ${cName} missing deps: ${missing.join(', ')}`);  continue; } }
-
-        // 2. Logic A: No previous run
-        if (!storedStatus) { logger.log('INFO', `[Versioning] ${cName}: New run needed (No prior status).`); calcsToAttempt.push(calc); continue; }
-
-        // 3. Logic B: Hash Mismatch
-        if (typeof storedStatus === 'string' && currentHash && storedStatus !== currentHash) { logger.log('INFO', `[Versioning] ${cName}: Code Changed. (Old: ${storedStatus.substring(0,6)}... New: ${currentHash.substring(0,6)}...)`); calcsToAttempt.push(calc); continue; }
-
-        // 4. Logic C: Upgrade Legacy Boolean -> Hash
-        if (storedStatus === true && currentHash) { logger.log('INFO', `[Versioning] ${cName}: Upgrading legacy status to Hash.`); calcsToAttempt.push(calc); continue; }
-    }
-
-    if (!calcsToAttempt.length) return null;
-
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
-    };
-
-    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-    if (!rootData) { logger.log('INFO', `[DateRunner] Root data missing for ${dateStr}. Skipping.`); return null; }
-    const runnableCalcs = calcsToAttempt.filter(c => checkRootDependencies(c, rootData.status).canRun);
-    if (!runnableCalcs.length) return null;
-    const standardToRun = runnableCalcs.filter(c => c.type === 'standard');
-    const metaToRun = runnableCalcs.filter(c => c.type === 'meta');
-    logger.log('INFO', `[DateRunner] Running ${dateStr}: ${standardToRun.length} std, ${metaToRun.length} meta`);
-    const dateUpdates = {};
-
-    try {
-        const calcsRunning = [...standardToRun, ...metaToRun];
-        const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-        const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr     = prevDate.toISOString().slice(0, 10);
-        const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
-
-        if (standardToRun.length) { const updates = await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false);
-            Object.assign(dateUpdates, updates); }
-
-        if (metaToRun.length) { const updates = await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false);
-            Object.assign(dateUpdates, updates); }
-
-    } catch (err) {
-        logger.log('ERROR', `[DateRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-        [...standardToRun, ...metaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
-        throw err; 
-    }
-
-    if (Object.keys(dateUpdates).length > 0) { await updateComputationStatus(dateStr, dateUpdates, config, dependencies); }
-
-    return { date: dateStr, updates: dateUpdates };
-}
-
-module.exports = { runComputationPass, runDateComputation };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,352 +0,0 @@
-/**
- * FILENAME: computation-system/helpers/orchestration_helpers.js
- * FEATURE: Dynamic Auto-Sharding (Transparent 1MB Limit Handling)
- * * DESCRIPTION:
- * This module orchestrates the execution of computations. It handles:
- * 1. Data Availability Checks
- * 2. Dependency Injection (fetching results from previous passes)
- * 3. Transparent Auto-Sharding:
- * - Writes: Automatically detects if a result > 900KB. Splits it into a '_shards' subcollection.
- * - Reads: Automatically detects sharded pointers and re-assembles the data.
- */
-
-const { ComputationController }              = require('../controllers/computation_controller');
-const { batchStoreSchemas }                  = require('../utils/schema_capture');
-const { normalizeName, commitBatchInChunks } = require('../utils/utils');
-const {
-    getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights,
-    getHistoryPartRefs, streamPortfolioData, streamHistoryData,
-    getRelevantShardRefs, loadDataByRefs
-} = require('../utils/data_loader');
-const mathLayer = require('../layers/index.js');
-const pLimit = require('p-limit');
-
-// Mappings for backward compatibility
-const LEGACY_MAPPING = {
-    DataExtractor: 'extract', HistoryExtractor: 'history', MathPrimitives: 'compute', Aggregators: 'aggregate', Validators: 'validate', SignalPrimitives: 'signals', SCHEMAS: 'schemas', DistributionAnalytics: 'distribution', TimeSeries: 'TimeSeries', priceExtractor: 'priceExtractor', InsightsExtractor: 'insights', UserClassifier: 'classifier', CognitiveBiases: 'bias', SkillAttribution: 'skill', Psychometrics: 'psychometrics'
-};
-
-function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
-
-function validateResultPatterns(logger, calcName, results, category) {
-    if (category === 'speculator' || category === 'speculators') return;
-    const tickers = Object.keys(results); const totalItems = tickers.length; if (totalItems < 5) return;
-    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object'); if (!sampleTicker) return;
-    Object.keys(results[sampleTicker]).forEach(key => {
-        if (key.startsWith('_')) return;
-        let nullCount = 0, nanCount = 0, undefinedCount = 0;
-        for (const t of tickers) { const val = results[t][key]; if (val === null) nullCount++; if (val === undefined) undefinedCount++; if (typeof val === 'number' && isNaN(val)) nanCount++; }
-        if (nanCount === totalItems) logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of items.`);
-        else if (undefinedCount === totalItems) logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of items.`);
-        else if (nullCount > (totalItems * 0.9)) logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
-    });
-}
-
-function checkRootDependencies(calcManifest, rootDataStatus) {
-    const missing = [];
-    if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
-    for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
-        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
-        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
-        else if (dep === 'price' && !rootDataStatus.hasPrices) missing.push('price');
-    }
-    return { canRun: missing.length === 0, missing };
-}
-
-async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger } = dependencies;
-    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false, insightsData = null, socialData = null;
-    try {
-        const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
-        if (dateToProcess >= earliestDates.price) { tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; })); }
-        await Promise.all(tasks);
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-        return { portfolioRefs, historyRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }, yesterdayPortfolioRefs: null };
-    } catch (err) { logger.log('ERROR', `Error checking data: ${err.message}`); return null; }
-}
-
-async function firestoreHelper(action, { key, updates, config, db }) {
-    const collections = { price: config.priceCollection || 'asset_prices', status: config.computationStatusCollection || 'computation_status', };
-    switch (action) {
-        case 'checkAvailability': try { const snapshot = await db.collection(collections.price).limit(1).get(); return !snapshot.empty; } catch (e) { return false; }
-        case 'fetchStatus': { if (!key) throw new Error('fetchStatus requires a key'); const docRef = db.collection(collections.status).doc(key); const snap = await docRef.get(); return snap.exists ? snap.data() : {}; }
-        case 'updateStatus': { if (!key) throw new Error('updateStatus requires a key'); if (!updates || Object.keys(updates).length === 0) return; const docRef = db.collection(collections.status).doc(key); await docRef.set(updates, { merge: true }); return true; }
-        default: throw new Error(`Unknown action: ${action}`);
-    }
-}
-
-async function checkPriceDataAvailability(config, dependencies) { return firestoreHelper('checkAvailability', { config, db: dependencies.db }); }
-async function fetchComputationStatus(dateStr, config, { db }) { return firestoreHelper('fetchStatus', { key: dateStr, config, db }); }
-async function fetchGlobalComputationStatus(config, { db }) { return firestoreHelper('fetchStatus', { key: 'global_status', config, db }); }
-async function updateComputationStatus(dateStr, updates, config, { db }) { return firestoreHelper('updateStatus', { key: dateStr, updates, config, db }); }
-
-/**
- * --- REFACTORED: fetchExistingResults ---
- * Transparently handles both standard documents and auto-sharded documents.
- * 1. Fetches the doc.
- * 2. Checks for `_sharded: true` flag.
- * 3. If sharded, fetches subcollection and merges data back into a single object.
- */
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const calcsToFetch = new Set();
-    for (const calc of calcsInPass) { if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); if (includeSelf && calc.isHistorical) calcsToFetch.add(normalizeName(calc.name)); }
-    if (!calcsToFetch.size) return {};
-    const fetched = {};
-    const docRefs = [];
-    const names   = [];
-
-    // 1. Prepare Reads
-    for (const name of calcsToFetch) {
-        const m = manifestMap.get(name);
-        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(m.category || 'unknown').collection(config.computationsSubcollection).doc(name)); names.push(name); }
-    }
-
-    if (docRefs.length) {
-        const snaps = await db.getAll(...docRefs);
-        const hydrationPromises = [];
-
-        // 2. Process Initial Snapshots
-        snaps.forEach((doc, i) => { const name = names[i]; if (!doc.exists) return; const data = doc.data(); if (data._sharded === true) { hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); } else if (data._completed) { fetched[name] = data; } }); // CHECK FOR AUTO-SHARDING FLAG
-        
-
-        // 3. Hydrate Sharded Data in Parallel
-        if (hydrationPromises.length > 0) { const hydratedResults = await Promise.all(hydrationPromises); hydratedResults.forEach(res => { fetched[res.name] = res.data; }); }
-    }
-    return fetched;
-}
-
-/**
- * Helper: Fetches all docs in the '_shards' subcollection and merges them.
- */
-async function hydrateAutoShardedResult(docRef, resultName) {
-    // Determine subcollection name (defaulting to '_shards')
-    const shardsCol = docRef.collection('_shards');
-    const snapshot = await shardsCol.get();
-    const assembledData = { _completed: true }; // Rebuild the object
-    snapshot.forEach(doc => { const chunk = doc.data(); Object.assign(assembledData, chunk); });
-    // Remove internal flags if they leaked into the shards
-    delete assembledData._sharded;
-    delete assembledData._completed;
-    return { name: resultName, data: assembledData };
-}
-
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
-    const { logger } = deps;
-    const controller = new ComputationController(config, deps);
-    const calcs = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
-    if (streamingCalcs.length === 0) return;
-    
-    logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
-    await controller.loader.loadMappings();
-    const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-    const prevDateStr = prevDate.toISOString().slice(0, 10);
-    const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
-    const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
-
-    let yP_chunk = {}, tH_chunk = {};
-    for await (const tP_chunk of tP_iter) {
-        if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
-        if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-        const promises = streamingCalcs.map(calc => controller.executor.executePerUser(calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, fetchedDeps, previousFetchedDeps));
-        await Promise.all(promises);
-    }
-    logger.log('INFO', `[${passName}] Streaming complete.`);
-}
-
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
-    const dStr = date.toISOString().slice(0, 10);
-    const logger = deps.logger;
-    const fullRoot = { ...rootData };
-    if (calcs.some(c => c.isHistorical)) {
-        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
-        const prevStr = prev.toISOString().slice(0, 10);
-        fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
-    }
-    const state = {};
-    for (const c of calcs) { try { const inst = new c.class(); inst.manifest = c; state[normalizeName(c.name)] = inst; logger.log('INFO', `${c.name} calculation running for ${dStr}`); } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } }
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
-    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
-}
-
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
-    const controller = new ComputationController(config, deps);
-    const dStr = date.toISOString().slice(0, 10);
-    const state = {};
-    for (const mCalc of calcs) {
-        try {
-            deps.logger.log('INFO', `${mCalc.name} calculation running for ${dStr}`);
-            const inst = new mCalc.class(); inst.manifest = mCalc;
-            await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
-            state[normalizeName(mCalc.name)] = inst;
-        } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
-    }
-    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
-}
-
-/**
- * --- REFACTORED: commitResults ---
- * Automatically detects result size.
- * If > 900KB, it splits the result into chunks and writes to a subcollection.
- * If < 900KB, it writes normally.
- */
-async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
-    const successUpdates = {};
-    const schemas = [];
-    for (const name in stateObj) {
-        const calc = stateObj[name];
-        let hasData = false;
-        try {
-            const result = await calc.getResult();
-            if (!result) { deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`); continue; }
-            const mainDocRef = deps.db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-            // AUTO-SHARDING LOGIC
-            const updates = await prepareAutoShardedWrites(result, mainDocRef, deps.logger);
-            // Collect Schemas if present
-            if (calc.manifest.class.getSchema) { 
-                const { class: _cls, ...safeMetadata } = calc.manifest; 
-                schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata }); 
-            }
-            if (updates.length > 0) {
-                await commitBatchInChunks(config, deps, updates, `${name} Results`);
-                successUpdates[name] = calc.manifest.hash || true;
-                const isSharded = updates.some(u => u.data._sharded === true);
-                deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written ${isSharded ? 'Sharded' : 'Standard'})`);
-            } else { 
-                deps.logger.log('INFO', `${name} for ${dStr}: - Empty Data`); 
-            }
-        } catch (e) { deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`); }
-    }
-    if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => { });
-    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
-        await updateComputationStatus(dStr, successUpdates, config, deps);
-        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
-    }
-    return successUpdates;
-}
-
-/**
- * Accurately calculates the size of a value according to Firestore storage rules.
- * Reference: https://firebase.google.com/docs/firestore/storage-size
- */
-function calculateFirestoreBytes(value) {
-    if (value === null) return 1;
-    if (value === undefined) return 0; // Firestore drops undefined fields
-    if (typeof value === 'boolean') return 1;
-    if (typeof value === 'number') return 8; // All numbers are 64-bit doubles or integers
-    if (typeof value === 'string') return Buffer.byteLength(value, 'utf8') + 1;
-    if (value instanceof Date) return 8; // Timestamps are 8 bytes
-    if (value.constructor && value.constructor.name === 'DocumentReference') { return Buffer.byteLength(value.path, 'utf8') + 16; }
-    if (Array.isArray(value)) { let sum = 0; for (const item of value) sum += calculateFirestoreBytes(item); return sum; }
-    // Handle Objects (Maps): Sum of (Key + 1 + Value)
-    if (typeof value === 'object') { let sum = 0; for (const k in value) { if (Object.prototype.hasOwnProperty.call(value, k)) { sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]); } } return sum; }
-    return 0; // Fallback
-}
-
-
-async function prepareAutoShardedWrites(result, docRef, logger) {
-    const SAFETY_THRESHOLD_BYTES = 1000 * 1024; // 1MB Limit (We target just under this)
-    const OVERHEAD_ALLOWANCE = 20 * 1024; // 20KB Safety margin for document path & metadata
-    const CHUNK_LIMIT = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
-    const totalSize = calculateFirestoreBytes(result); // 1. Calculate Total Size Once (O(N))
-    const docPathSize = Buffer.byteLength(docRef.path, 'utf8') + 16; // Add the size of the document path itself (Firestore counts this against the 1MB limit)
-    if ((totalSize + docPathSize) < CHUNK_LIMIT) { const data = { ...result, _completed: true, _sharded: false }; return [{ ref: docRef, data, options: { merge: true } }]; } // CASE A: Fits in one document
-    logger.log('INFO', `[AutoShard] Result size ~${Math.round(totalSize/1024)}KB exceeds limit. Sharding...`);
-    const writes          = [];
-    const shardCollection = docRef.collection('_shards');
-    let currentChunk      = {};
-    let currentChunkSize  = 0;
-    let shardIndex        = 0;
-    for (const [key, value] of Object.entries(result)) { // 2. Efficient O(N) Loop
-        if (key.startsWith('_')) continue;
-        const keySize = Buffer.byteLength(key, 'utf8') + 1;         // Calculate size of just this item
-        const valueSize = calculateFirestoreBytes(value);
-        const itemSize = keySize + valueSize;
-        if (currentChunkSize + itemSize > CHUNK_LIMIT) { // Check if adding this item would overflow the current chunk
-            // Flush current chunk
-            writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); // Overwrite
-            shardIndex++;
-            currentChunk = {};
-            currentChunkSize = 0;
-        }
-        // Add to current chunk
-        currentChunk[key] = value;
-        currentChunkSize += itemSize;
-    }
-    // Flush final chunk
-    if (Object.keys(currentChunk).length > 0) { writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } }); }
-    const pointerData = { _completed: true,  _sharded: true, _shardCount: shardIndex + 1, _lastUpdated: new Date().toISOString() }; // Pointer Document
-    writes.push({ ref: docRef, data: pointerData, options: { merge: false } }); // Use merge: false to ensure we overwrite any previous non-sharded blob
-    return writes;
-}
-
-async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps;
-    const controller = new ComputationController(config, deps);
-    const mappings = await controller.loader.loadMappings();
-    let targetInstrumentIds = [];
-    if (targetTickers && targetTickers.length > 0) {
-        const tickerToInst = mappings.tickerToInstrument || {};
-        targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } 
-    }
-    const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
-    if (!allShardRefs.length) { logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.'); return; }
-    const OUTER_CONCURRENCY_LIMIT = 2, SHARD_BATCH_SIZE = 20, WRITE_BATCH_LIMIT = 50;
-    logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
-    const shardChunks = []; for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
-    const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
-    const chunkPromises = [];
-    for (let index = 0; index < shardChunks.length; index++) {
-        const shardChunkRefs = shardChunks[index];
-        chunkPromises.push(outerLimit(async () => {
-            try {
-                logger.log('INFO', `[BatchPrice] Processing chunk ${index + 1}/${shardChunks.length} (${shardChunkRefs.length} shards)...`);
-                const pricesData = await loadDataByRefs(config, deps, shardChunkRefs);
-                if (targetInstrumentIds.length > 0) { const requestedSet = new Set(targetInstrumentIds); for (const loadedInstrumentId in pricesData) { if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } } }
-                const writes = [];
-                for (const dateStr of dateStrings) {
-                    const dynamicMathContext = {};
-                    for (const [key, value] of Object.entries(mathLayer)) { dynamicMathContext[key] = value; if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} }
-                    const context = { mappings, prices: { history: pricesData }, date: { today: dateStr }, math: dynamicMathContext };
-                    for (const calcManifest of calcs) {
-                        try {
-                            const instance = new calcManifest.class(); await instance.process(context); const result = await instance.getResult();
-                            if (result && Object.keys(result).length > 0) {
-                                let dataToWrite = result; if (result.by_instrument) dataToWrite = result.by_instrument;
-                                if (Object.keys(dataToWrite).length > 0) {
-                                    const docRef = db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category).collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-                                    writes.push({ ref: docRef, data: { ...dataToWrite, _completed: true }, options: { merge: true } });
-                                }
-                            }
-                        } catch (err) { logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`); }
-                    }
-                }
-                if (writes.length > 0) {
-                    const commitBatches = []; for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
-                    const commitLimit = pLimit(10);
-                    await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
-                        const batch = db.batch(); batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
-                        try { await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`); } catch (commitErr) { logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); }
-                    })));
-                }
-            } catch (chunkErr) { logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message }); }
-        }));
-    }
-    await Promise.all(chunkPromises);
-    logger.log('INFO', '[BatchPrice] Optimization pass complete.');
-}
-
-module.exports = { groupByPass, checkRootDependencies, checkRootDataAvailability, fetchExistingResults, fetchComputationStatus, fetchGlobalComputationStatus, updateComputationStatus, runStandardComputationPass, runMetaComputationPass, runBatchPriceComputation };