bulltrackers-module 1.0.198 → 1.0.200

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,8 +1,8 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
- * FIXED: TS Error (controller.loader.mappings)
- * ADDED: Smart Shard Lookup for specific tickers
- * OPTIMIZED: Added Concurrency (Parallel Commits & Pipelined Shards) for runBatchPriceComputation
+ * FIXED: Context math mapping in runBatchPriceComputation (resolves 'undefined' crash).
+ * IMPROVED: Explicit logging for every calculation run (Start, Success, Failure).
+ * OPTIMIZED: Parallel Commits & Pipelined Shards.
  */
 
 const { ComputationController }              = require('../controllers/computation_controller');
@@ -14,7 +14,14 @@ const {
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
 
-const pLimit = require('p-limit'); // TODO, THIS OUGHT TO BE INJECTED.
+// --- FIX 1: Import Math Layer Primitives for Correct Context Mapping ---
+const { 
+    DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, 
+    Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, 
+    TimeSeries, priceExtractor 
+} = require('../layers/math_primitives.js');
+
+const pLimit = require('p-limit'); 
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -22,29 +29,23 @@ const pLimit = require('p-limit'); // TODO, THIS OUGHT TO BE INJECTED.
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
 /**
- * --- NEW HELPER: PASSIVE DATA VALIDATION ---
+ * --- PASSIVE DATA VALIDATION ---
  * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
- * Logs warnings but DOES NOT block the commit.
  */
-function validateResultPatterns(logger, calcName, results, category) { // TODO, THIS COULD BE MUCH MORE SOPHISTICATED, WE WILL NEVER FORCE FAIL A COMPUTATION REGARDLESS, BUT IT COULD BE A GREAT WARNING SYSTEM, USE GLOUD LOG SINKS TO DETECT PROBLEMS BASED ON THESE LOGS
-    // 1. Skip Speculators (Too sparse, nulls are expected)
+function validateResultPatterns(logger, calcName, results, category) { 
     if (category === 'speculator' || category === 'speculators') return;
 
     const tickers = Object.keys(results);
     const totalItems = tickers.length;
     
-    // 2. Need a decent sample size to judge patterns
     if (totalItems < 5) return; 
 
-    // 3. Get all keys from the first valid object
-    // We assume schema is roughly consistent across tickers
     const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
     if (!sampleTicker) return;
     
     const keys = Object.keys(results[sampleTicker]);
     
     keys.forEach(key => {
-        // Skip internal keys or metadata
         if (key.startsWith('_')) return;
 
         let nullCount = 0;
@@ -58,14 +59,11 @@ function validateResultPatterns(logger, calcName, results, category) { // TODO,
             if (typeof val === 'number' && isNaN(val)) nanCount++;
         }
 
-        // 4. Define Thresholds
-        // If 100% of data is NaN or Undefined, that's almost certainly a bug.
         if (nanCount === totalItems) {
             logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items. Code bug likely.`);
         } else if (undefinedCount === totalItems) {
             logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items. Code bug likely.`);
         } 
-        // 5. Nulls are tricky. warn if >90%, but don't error (might be valid logic like "no shorts")
         else if (nullCount > (totalItems * 0.9)) {
             logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items. Check logic if this is unexpected.`);
         }
@@ -329,10 +327,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             }
 
             if (Object.keys(standardRes).length) {
-                // --- NEW: Run Passive Validation ---
-                // We do this BEFORE marking it completed, but we do NOT stop the write.
                 validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
-                // -----------------------------------
 
                 standardRes._completed = true;
                 writes.push({
@@ -388,10 +383,10 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
  * OPTIMIZED: Implements concurrency for both Shard Processing and Write Commits
  */
 async function runBatchPriceComputation(config, deps, dateStrings, calcs, targetTickers = []) {
-    const { logger, db, calculationUtils } = deps; // Ensure calculationUtils is available for retry
+    const { logger, db, calculationUtils } = deps; 
     const controller = new ComputationController(config, deps);
     
-    // 1. FIX: Call loadMappings() correctly and get the result
+    // 1. Call loadMappings() correctly and get the result
     const mappings = await controller.loader.loadMappings(); 
     
     // 2. Resolve Shards (All or Subset)
@@ -413,12 +408,9 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
     }
 
     // 3. Execution Planning
-    // CONCURRENCY SETTING:
-    // Limit outer concurrency (processing shard chunks) to 2 to prevent contention on daily result docs.
-    // While Firestore handles concurrent writes to the same doc, limiting this avoids excessive retries.
     const OUTER_CONCURRENCY_LIMIT = 2; 
     const SHARD_BATCH_SIZE = 20;
-    const WRITE_BATCH_LIMIT = 50; // Keep write batch size small (payload safety)
+    const WRITE_BATCH_LIMIT = 50; 
 
     logger.log('INFO', `[BatchPrice] Execution Plan: ${dateStrings.length} days, ${allShardRefs.length} shards. Concurrency: ${OUTER_CONCURRENCY_LIMIT}.`);
 
@@ -440,7 +432,6 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
             // Optional Filtering for Subset Mode
             if (targetInstrumentIds.length > 0) {
                 const requestedSet = new Set(targetInstrumentIds);
-                // Iterate over the loaded data and delete anything we didn't ask for
                 for (const loadedInstrumentId in pricesData) {
                     if (!requestedSet.has(loadedInstrumentId)) {
                         delete pricesData[loadedInstrumentId];
@@ -450,26 +441,44 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
             const writes = [];
             
             // --- CALCULATION PHASE ---
-            // This builds up the array of writes (one per date)
             for (const dateStr of dateStrings) {
+                // --- FIX 2: Manually map math primitives to their alias names ---
+                // This matches the ContextBuilder logic in ComputationController
+                // and fixes the "Cannot read properties of undefined (reading 'standardDeviation')" error.
                 const context = {
                     mappings,
                     prices: { history: pricesData },
                     date: { today: dateStr },
-                    math: require('../layers/math_primitives.js')
+                    math: { 
+                        extract: DataExtractor, 
+                        history: HistoryExtractor, 
+                        compute: MathPrimitives, 
+                        aggregate: Aggregators, 
+                        validate: Validators, 
+                        signals: SignalPrimitives, 
+                        schemas: SCHEMAS, 
+                        distribution : DistributionAnalytics, 
+                        TimeSeries: TimeSeries, 
+                        priceExtractor : priceExtractor 
+                    }
                 };
 
                 for (const calcManifest of calcs) {
                     try {
+                        // --- LOGGING FIX: Log start of calculation ---
+                        logger.log('INFO', `[BatchPrice] >> Running ${calcManifest.name} for ${dateStr}...`);
+
                         const instance = new calcManifest.class();
                         await instance.process(context);
                         const result = await instance.getResult();
                         
+                        let hasContent = false;
                         if (result && Object.keys(result).length > 0) {
                             let dataToWrite = result;
                             if (result.by_instrument) dataToWrite = result.by_instrument; 
                             
                             if (Object.keys(dataToWrite).length > 0) {
+                                hasContent = true;
                                 const docRef = db.collection(config.resultsCollection).doc(dateStr)
                                     .collection(config.resultsSubcollection).doc(calcManifest.category)
                                     .collection(config.computationsSubcollection).doc(normalizeName(calcManifest.name));
@@ -481,22 +490,28 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
                                 });
                             }
                         }
+
+                        // --- LOGGING FIX: Log success/completion ---
+                        if (hasContent) {
+                            logger.log('INFO', `[BatchPrice] \u2714 Finished ${calcManifest.name} for ${dateStr}. Found data.`);
+                        } else {
+                            logger.log('INFO', `[BatchPrice] - Finished ${calcManifest.name} for ${dateStr}. No result data.`);
+                        }
+
                     } catch (err) {
-                        logger.log('ERROR', `[BatchPrice] Calc ${calcManifest.name} failed for ${dateStr}`, { error: err.message });
+                        // --- LOGGING FIX: Explicit failure log ---
+                        logger.log('ERROR', `[BatchPrice] \u2716 Failed ${calcManifest.name} for ${dateStr}: ${err.message}`);
                     }
                 }
             }
             
             // --- PARALLEL COMMIT PHASE ---
-            // Instead of committing sequentially via commitBatchInChunks, we process these writes in parallel.
-            // Since each write targets a DIFFERENT date (different document), parallelizing this is safe and fast.
             if (writes.length > 0) {
                 const commitBatches = [];
                 for (let i = 0; i < writes.length; i += WRITE_BATCH_LIMIT) {
                     commitBatches.push(writes.slice(i, i + WRITE_BATCH_LIMIT));
                 }
 
-                // Use a higher concurrency for commits since they target disjoint documents
                 const commitLimit = pLimit(10); 
                 
                 await Promise.all(commitBatches.map((batchWrites, bIndex) => commitLimit(async () => {
@@ -507,7 +522,6 @@ async function runBatchPriceComputation(config, deps, dateStrings, calcs, target
                         await calculationUtils.withRetry(() => batch.commit(), `BatchPrice-C${index}-B${bIndex}`);
                     } catch (commitErr) {
                         logger.log('ERROR', `[BatchPrice] Commit failed for Chunk ${index} Batch ${bIndex}.`, { error: commitErr.message });
-                        // We log but don't throw, to allow other batches to succeed
                     }
                 })));
             }

package/functions/generic-api/helpers/api_helpers.js

@@ -131,6 +131,7 @@ const createApiHandler = (config, dependencies, calcMap) => {
             const computationKeys = req.query.computations.split(',');
             const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
             const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
+            res.set('Cache-Control', 'public, max-age=300, s-maxage=3600');
             res.status(200).send({
                 status: 'success',
                 metadata: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.198",
+  "version": "1.0.200",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [