bulltrackers-module 1.0.193 → 1.0.194

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -20,6 +20,57 @@ const pLimit = require('p-limit'); // Ensure p-limit is required
  */
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
+/**
+ * --- NEW HELPER: PASSIVE DATA VALIDATION ---
+ * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
+ * Logs warnings but DOES NOT block the commit.
+ */
+function validateResultPatterns(logger, calcName, results, category) {
+    // 1. Skip Speculators (Too sparse, nulls are expected)
+    if (category === 'speculator' || category === 'speculators') return;
+
+    const tickers = Object.keys(results);
+    const totalItems = tickers.length;
+    
+    // 2. Need a decent sample size to judge patterns
+    if (totalItems < 5) return; 
+
+    // 3. Get all keys from the first valid object
+    // We assume schema is roughly consistent across tickers
+    const sampleTicker = tickers.find(t => results[t] && typeof results[t] === 'object');
+    if (!sampleTicker) return;
+    
+    const keys = Object.keys(results[sampleTicker]);
+    
+    keys.forEach(key => {
+        // Skip internal keys or metadata
+        if (key.startsWith('_')) return;
+
+        let nullCount = 0;
+        let nanCount = 0;
+        let undefinedCount = 0;
+
+        for (const t of tickers) {
+            const val = results[t][key];
+            if (val === null) nullCount++;
+            if (val === undefined) undefinedCount++;
+            if (typeof val === 'number' && isNaN(val)) nanCount++;
+        }
+
+        // 4. Define Thresholds
+        // If 100% of data is NaN or Undefined, that's almost certainly a bug.
+        if (nanCount === totalItems) {
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is NaN for 100% of ${totalItems} items. Code bug likely.`);
+        } else if (undefinedCount === totalItems) {
+            logger.log('ERROR', `[DataQuality] Calc '${calcName}' field '${key}' is UNDEFINED for 100% of ${totalItems} items. Code bug likely.`);
+        } 
+        // 5. Nulls are tricky. warn if >90%, but don't error (might be valid logic like "no shorts")
+        else if (nullCount > (totalItems * 0.9)) {
+            logger.log('WARN', `[DataQuality] Calc '${calcName}' field '${key}' is NULL for ${nullCount}/${totalItems} items. Check logic if this is unexpected.`);
+        }
+    });
+}
+
 /**
  * Checks if all root data dependencies for a given calculation are met.
  */
@@ -277,6 +328,11 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             }
 
             if (Object.keys(standardRes).length) {
+                // --- NEW: Run Passive Validation ---
+                // We do this BEFORE marking it completed, but we do NOT stop the write.
+                validateResultPatterns(deps.logger, name, standardRes, calc.manifest.category);
+                // -----------------------------------
+
                 standardRes._completed = true;
                 writes.push({
                     ref: deps.db.collection(config.resultsCollection).doc(dStr)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.193",
+  "version": "1.0.194",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [