bulltrackers-module 1.0.214 → 1.0.215

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -2,6 +2,7 @@
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
  * FIXED: 'commitResults' now records the CODE HASH in the status document
  * instead of a boolean, enabling auto-invalidation on code changes.
+ * UPDATED: Dynamic math context loading from 'layers/index.js' to support modular layers.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -13,14 +14,32 @@ const {
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
-const {
-    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators,
-    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics,
-    TimeSeries, priceExtractor
-} = require('../layers/math_primitives.js');
+// --- DYNAMIC LAYER LOADING ---
+// Replaces the old static import from 'math_primitives.js'
+const mathLayer = require('../layers/index.js');
 
 const pLimit = require('p-limit');
 
+// Mappings to ensure backward compatibility with existing calculation code
+// (e.g. allowing 'math.compute' to resolve to 'MathPrimitives')
+const LEGACY_MAPPING = {
+    DataExtractor:         'extract',
+    HistoryExtractor:      'history',
+    MathPrimitives:        'compute',
+    Aggregators:           'aggregate',
+    Validators:            'validate',
+    SignalPrimitives:      'signals',
+    SCHEMAS:               'schemas',
+    DistributionAnalytics: 'distribution',
+    TimeSeries:            'TimeSeries',
+    priceExtractor:        'priceExtractor',
+    InsightsExtractor:     'insights',
+    UserClassifier:        'classifier',
+    CognitiveBiases:       'bias',
+    SkillAttribution:      'skill',
+    Psychometrics:         'psychometrics'
+};
+
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
 /**
@@ -28,39 +47,26 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
  */
 function validateResultPatterns(logger, calcName, results, category) {
     if (category === 'speculator' || category === 'speculators') return;
-
     const tickers    = Object.keys(results);
     const totalItems = tickers.length;
-
     if (totalItems < 5) return;
-
     const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
     if (!sampleTicker) return;
-
     const keys = Object.keys(results[sampleTicker]);
-
     keys.forEach(key => {
         if (key.startsWith('_')) return;
-
         let nullCount      = 0;
         let nanCount       = 0;
         let undefinedCount = 0;
-
         for (const t of tickers) {
             const val = results[t][key];
             if (val === null) nullCount++;
             if (val === undefined) undefinedCount++;
             if (typeof val === 'number' && isNaN(val)) nanCount++;
         }
-
-        if (nanCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
-        } else if (undefinedCount === totalItems) {
-            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`);
-        }
-        else if (nullCount > (totalItems * 0.9)) {
-            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`);
-        }
+        if (nanCount === totalItems) { logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items.`);
+        } else if (undefinedCount === totalItems) { logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items.`); }
+        else if (nullCount > (totalItems * 0.9)) { logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items.`); }
     });
 }
 
@@ -81,8 +87,7 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData = null, socialData = null;
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false, insightsData = null, socialData = null;
 
     try {
         const tasks = [];
@@ -91,12 +96,8 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
         if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
 
-        if (dateToProcess >= earliestDates.price) {
-            tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
-        }
-
+        if (dateToProcess >= earliestDates.price) { tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; })); }
         await Promise.all(tasks);
-
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
 
         return {
@@ -105,97 +106,55 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             todayInsights: insightsData,
             todaySocialPostInsights: socialData,
             status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices },
-            yesterdayPortfolioRefs: null // Will be populated if needed
+            yesterdayPortfolioRefs: null
         };
 
-    } catch (err) {
-        logger.log('ERROR', `Error checking data: ${err.message}`);
-        return null;
-    }
+    } catch (err) { logger.log('ERROR', `Error checking data: ${err.message}`); return null; }
 }
 
-async function checkPriceDataAvailability(config, dependencies) {
-    const { db } = dependencies;
-    const collection = config.priceCollection || 'asset_prices';
-    try {
-        const snapshot = await db.collection(collection).limit(1).get();
-        if (snapshot.empty) return false;
-        return true;
-    } catch (e) {
-        return false;
+async function firestoreHelper(action, { key, updates, config, db }) {
+  const collections = { price: config.priceCollection || 'asset_prices', status: config.computationStatusCollection || 'computation_status', };
+
+  switch (action) {
+    case 'checkAvailability': {
+      try { const snapshot = await db.collection(collections.price).limit(1).get(); return !snapshot.empty; } catch (e) { return false; } }
+
+    case 'fetchStatus': {
+      if (!key) throw new Error('fetchStatus requires a key');
+      const docRef = db.collection(collections.status).doc(key);
+      const snap = await docRef.get();
+      return snap.exists ? snap.data() : {};
     }
-}
 
-async function fetchComputationStatus(dateStr, config, { db }) {
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc(dateStr);
-    const snap = await docRef.get();
-    return snap.exists ? snap.data() : {};
-}
+    case 'updateStatus': {
+      if (!key) throw new Error('updateStatus requires a key');
+      if (!updates || Object.keys(updates).length === 0) return;
+      const docRef = db.collection(collections.status).doc(key);
+      await docRef.set(updates, { merge: true });
+      return true;
+    }
 
-async function fetchGlobalComputationStatus(config, { db }) {
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc('global_status');
-    const snap       = await docRef.get();
-    return snap.exists ? snap.data() : {};
+    default: throw new Error(`Unknown action: ${action}`); }
 }
 
-async function updateComputationStatus(dateStr, updates, config, { db }) {
-    if (!updates || Object.keys(updates).length === 0) return;
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc(dateStr);
-    await docRef.set(updates, { merge: true });
-}
+async function checkPriceDataAvailability    (config, dependencies)             { return firestoreHelper('checkAvailability', { config, db: dependencies.db }); }
+async function fetchComputationStatus        (dateStr, config, { db })          { return firestoreHelper('fetchStatus', { key: dateStr, config, db }); }
+async function fetchGlobalComputationStatus  (config, { db })                   { return firestoreHelper('fetchStatus', { key: 'global_status', config, db }); }
+async function updateComputationStatus       (dateStr, updates, config, { db }) { return firestoreHelper('updateStatus', { key: dateStr, updates, config, db }); }
 
-async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
-    if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
-    const collection = config.computationStatusCollection || 'computation_status';
-    const docRef     = db.collection(collection).doc('global_status');
-    const flattenUpdates = {};
-    for (const [date, statuses] of Object.entries(updatesByDate)) {
-        for (const [calc, status] of Object.entries(statuses)) {
-            flattenUpdates[`${date}.${calc}`] = status;
-        }
-    }
-    try {
-        await docRef.update(flattenUpdates);
-    } catch (err) {
-        if (err.code === 5) {
-            const deepObj = {};
-            for (const [date, statuses] of Object.entries(updatesByDate)) {
-                deepObj[date] = statuses;
-            }
-            await docRef.set(deepObj, { merge: true });
-        } else {
-            throw err;
-        }
-    }
-}
 
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
-    for (const calc of calcsInPass) {
-        if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
-        if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
-    }
+    for (const calc of calcsInPass) { if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); } if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); } }
     if (!calcsToFetch.size) return {};
     const fetched = {};
     const docRefs = [];
     const names   = [];
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) {
-            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection).doc(name));
-            names.push(name);
-        }
-    }
-    if (docRefs.length) {
-        const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } });
-    }
+        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr) .collection(config.resultsSubcollection).doc(m.category || 'unknown') .collection(config.computationsSubcollection).doc(name)); names.push(name); } }
+    if (docRefs.length) { const snaps = await db.getAll(...docRefs); snaps.forEach((doc, i) => { if (doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data(); } }); }
     return fetched;
 }
 
@@ -203,10 +162,7 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const { logger } = deps;
     const controller     = new ComputationController(config, deps);
     const calcs          = Object.values(state).filter(c => c && c.manifest);
-    const streamingCalcs = calcs.filter(c =>
-        c.manifest.rootDataDependencies.includes('portfolio') ||
-        c.manifest.rootDataDependencies.includes('history')
-    );
+    const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history') );
 
     if (streamingCalcs.length === 0) return;
 
@@ -258,12 +214,7 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 
     const state = {};
     for (const c of calcs) {
-        try {
-            const inst = new c.class();
-            inst.manifest = c;
-            state[normalizeName(c.name)] = inst;
-            logger.log('INFO', `${c.name} calculation running for ${dStr}`);
-        }
+        try { const inst = new c.class(); inst.manifest = c; state[normalizeName(c.name)] = inst; logger.log('INFO', `${c.name} calculation running for ${dStr}`); }
         catch (e) { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
@@ -305,10 +256,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
         try {
             const result = await calc.getResult();
-            if (!result) {
-                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`);
-                continue;
-            }
+            if (!result) { deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`); continue; }
 
             const standardRes   = {};
             const shardedWrites = [];
@@ -320,65 +268,33 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     const sData = result[key];
                     for (const colName in sData) {
                         const docsMap = sData[colName];
-                        for (const docId in docsMap) {
-                            const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId);
-                            shardedWrites.push({
-                                ref,
-                                data: { ...docsMap[docId], _completed: true }
-                            });
-                        }
-                    }
+                        for (const docId in docsMap) { const ref = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(colName).doc(docId); shardedWrites.push({ ref, data: { ...docsMap[docId], _completed: true } }); } }
                     if (Object.keys(sData).length > 0) hasData = true;
-                } else {
-                    standardRes[key] = result[key];
-                }
+                } else { standardRes[key] = result[key]; }
             }
 
             // 2. Prepare Standard Result Write
             if (Object.keys(standardRes).length) {
                 validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
                 standardRes._completed = true;
-
-                const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                    .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                    .collection(config.computationsSubcollection).doc(name);
-
-                calcWrites.push({
-                    ref: docRef,
-                    data: standardRes
-                });
+                const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(calc.manifest.category) .collection(config.computationsSubcollection).doc(name);
+                calcWrites.push({ ref: docRef, data: standardRes });
                 hasData = true;
             }
 
             // 3. Queue Schema (Safe to accumulate)
-            if (calc.manifest.class.getSchema) {
-                const { class: _cls, ...safeMetadata } = calc.manifest;
-                schemas.push({
-                    name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata
-                });
-            }
+            if (calc.manifest.class.getSchema) { const { class: _cls, ...safeMetadata } = calc.manifest; schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: safeMetadata }); }
 
             // 4. ATTEMPT COMMIT FOR THIS CALCULATION ONLY
             if (hasData) {
                 const allWritesForCalc = [...calcWrites, ...shardedWrites];
-
                 if (allWritesForCalc.length > 0) {
                     await commitBatchInChunks(config, deps, allWritesForCalc, `${name} Results`);
-
-                    // --- CRITICAL UPDATE: Store the Smart Hash ---
                     successUpdates[name] = calc.manifest.hash || true;
-                    
                     deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written)`);
-                } else {
-                    deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`);
-                }
-            } else {
-                deps.logger.log('INFO', `${name} for ${dStr}: - Empty`);
-            }
-
-        } catch (e) {
-            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
-        }
+                } else { deps.logger.log('INFO', `${name} for ${dStr}: - No Data to Write`); }
+            } else { deps.logger.log('INFO', `${name} for ${dStr}: - Empty`); }
+        } catch (e) { deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`); }
     }
 
     // Save Schemas (Best effort, isolated)
@@ -405,29 +321,20 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
     if (targetTickers && targetTickers.length > 0) {
         const tickerToInst = mappings.tickerToInstrument || {};
         targetInstrumentIds = targetTickers.map(t => tickerToInst[t]).filter(id => id);
-        if (targetInstrumentIds.length === 0) {
-            logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.');
-            return;
-        }
-    }
+        if (targetInstrumentIds.length === 0) { logger.log('WARN', '[BatchPrice] Target tickers provided but no IDs found. Aborting.'); return; } }
 
     const allShardRefs = await getRelevantShardRefs(config, deps, targetInstrumentIds);
 
-    if (!allShardRefs.length) {
-        logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.');
-        return;
-    }
+    if (!allShardRefs.length) { logger.log('WARN', '[BatchPrice] No relevant price shards found. Exiting.'); return; }
 
     const OUTER_CONCURRENCY_LIMIT = 2;
-    const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50;
+    const SHARD_BATCH_SIZE        = 20;
+    const WRITE_BATCH_LIMIT       = 50;
 
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
 
     const shardChunks = [];
-    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) {
-        shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE));
-    }
+    for (let i = 0; i < allShardRefs.length; i += SHARD_BATCH_SIZE) { shardChunks.push(allShardRefs.slice(i, i + SHARD_BATCH_SIZE)); }
 
     const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
 
@@ -442,32 +349,22 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
 
                 if (targetInstrumentIds.length > 0) {
                     const requestedSet = new Set(targetInstrumentIds);
-                    for (const loadedInstrumentId in pricesData) {
-                        if (!requestedSet.has(loadedInstrumentId)) {
-                            delete pricesData[loadedInstrumentId];
-                        }
-                    }
+                    for (const loadedInstrumentId in pricesData) { if (!requestedSet.has(loadedInstrumentId)) { delete pricesData[loadedInstrumentId]; } }
                 }
 
                 const writes = [];
 
                 for (const dateStr of dateStrings) {
+                    
+                    // --- DYNAMIC MATH CONTEXT CONSTRUCTION ---
+                    const dynamicMathContext = {};
+                    for (const [key, value] of Object.entries(mathLayer)) { dynamicMathContext[key] = value;  if (LEGACY_MAPPING[key]) { dynamicMathContext[LEGACY_MAPPING[key]] = value;} }
+
                     const context = {
                         mappings,
                         prices: { history: pricesData },
                         date: { today: dateStr },
-                        math: {
-                            extract: DataExtractor,
-                            history: HistoryExtractor,
-                            compute: MathPrimitives,
-                            aggregate: Aggregators,
-                            validate: Validators,
-                            signals: SignalPrimitives,
-                            schemas: SCHEMAS,
-                            distribution: DistributionAnalytics,
-                            TimeSeries: TimeSeries,
-                            priceExtractor: priceExtractor
-                        }
+                        math: dynamicMathContext // Injected here
                     };
 
                     for (const calcManifest of calcs) {
@@ -481,28 +378,18 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
                                 if (result.by_instrument) dataToWrite = result.by_instrument;
 
                                 if (Object.keys(dataToWrite).length > 0) {
-                                    const docRef = db.collection(config.resultsCollection).doc(dateStr)
-                                        .collection(config.resultsSubcollection).doc(calcManifest.category)
-                                        .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
-
-                                    writes.push({
-                                        ref: docRef,
-                                        data: { ...dataToWrite, _completed: true },
-                                        options: { merge: true }
-                                    });
+                                    const docRef = db.collection(config.resultsCollection).doc(dateStr) .collection(config.resultsSubcollection).doc(calcManifest.category) .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
+
+                                    writes.push({ ref: docRef, data: { ...dataToWrite, _completed: true }, options: { merge: true } });
                                 }
                             }
-                        } catch (err) {
-                            logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
-                        }
+                        } catch (err) { logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`); }
                     }
                 }
 
                 if (writes.length > 0) {
                     const commitBatches = [];
-                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
-                        commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
-                    }
+                    for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) { commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT)); }
 
                     const commitLimit = pLimit(10);
 
@@ -510,17 +397,12 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
                         const batch = db.batch();
                         batchWrites.forEach(w => batch.set(w.ref, w.data, w.options));
 
-                        try {
-                            await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
-                        } catch (commitErr) {
-                            logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
-                        }
+                        try { await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
+                        } catch (commitErr) { logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message }); }
                     })));
                 }
 
-            } catch (chunkErr) {
-                logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message });
-            }
+            } catch (chunkErr) { logger.log('ERROR', `[BatchPrice] Fatal error processing Chunk ${index}.`, { error: chunkErr.message }); }
         }));
     }
 
@@ -536,7 +418,6 @@ module.exports = {
     fetchComputationStatus,
     fetchGlobalComputationStatus,
     updateComputationStatus,
-    updateGlobalComputationStatus,
     runStandardComputationPass,
     runMetaComputationPass,
     runBatchPriceComputation