aiden-shared-calculations-unified 1.0.30 → 1.0.32

package/calculations/behavioural/historical/asset_crowd_flow.js

@@ -82,7 +82,7 @@ class AssetCrowdFlow {
     async getResult() {
         if (this.user_count === 0 || !this.dates.today) {
             console.warn('[AssetCrowdFlow] No users processed or dates missing.');
-            return {};
+            return null; // <--- MODIFICATION
         }
 
         // Load priceMap and mappings if not loaded
@@ -94,9 +94,19 @@ class AssetCrowdFlow {
                 ]);
                 this.priceMap = priceData;
                 this.mappings = mappingData;
+
+                // --- START NEW CHECK ---
+                // If priceData failed to load (e.g., returned {} from provider)
+                // this.priceMap will be empty. We must abort.
+                if (!this.priceMap || Object.keys(this.priceMap).length === 0) {
+                    console.error('[AssetCrowdFlow] CRITICAL: Price map is empty or failed to load. Aborting calculation to allow backfill.');
+                    return null; // Return null to trigger backfill
+                }
+                // --- END NEW CHECK ---
+
             } catch (err) {
                 console.error('[AssetCrowdFlow] Failed to load dependencies:', err);
-                return {};
+                return null; // <--- MODIFICATION: Return null on error
             }
         }
 
@@ -129,14 +139,8 @@ class AssetCrowdFlow {
             }
 
             if (priceChangePct === null) {
-                finalResults[ticker] = {
-                    net_crowd_flow_pct: 0,
-                    avg_value_day1: avg_day1_value,
-                    avg_value_day2: avg_day2_value,
-                    price_change_pct: null,
-                    user_sample_size: this.user_count,
-                    warning: 'Missing price data for calculation'
-                };
+                // <--- MODIFICATION: We no longer add a warning object. We just skip it.
+                // If it's missing price data, it won't be in finalResults.
                 continue;
             }
 
@@ -156,6 +160,15 @@ class AssetCrowdFlow {
             };
         }
 
+        // --- START NEW CHECK ---
+        // If all tickers were skipped due to missing price data,
+        // finalResults will be empty. Return null to trigger backfill.
+        if (Object.keys(finalResults).length === 0) {
+            console.warn(`[AssetCrowdFlow] No results generated for ${this.dates.today}. This likely means all price data was missing. Returning null for backfill.`);
+            return null;
+        }
+        // --- END NEW CHECK ---
+
         return finalResults;
     }
 
@@ -169,4 +182,4 @@ class AssetCrowdFlow {
     }
 }
 
-module.exports = AssetCrowdFlow;
+module.exports = AssetCrowdFlow;

package/calculations/meta/cash-flow-deployment.js

@@ -45,6 +45,18 @@ class CashFlowDeployment {
             if (snap.exists) dataMap.set(idx, snap.data());
         });
 
+        // --- START MODIFICATION ---
+        // Check for THIS DATE's dependencies (cash-flow and asset-flow) FIRST.
+        // These are at the end of the 'dates' array.
+        const cashFlowData = dataMap.get(this.lookbackDays);
+        const assetFlowData = dataMap.get(this.lookbackDays + 1);
+
+        if (!cashFlowData || !assetFlowData) {
+            logger.log('WARN', `[CashFlowDeployment] Missing critical dependency data (cash-flow or asset-flow) for ${dateStr}. Skipping to allow backfill.`);
+            return null; // This allows backfill
+        }
+        // --- END MODIFICATION ---
+
         // find deposit signal
         let depositSignal = null;
         let depositSignalDay = null;
@@ -77,14 +89,10 @@ class CashFlowDeployment {
             };
         }
 
-        // today's two dependencies are the last 2 refs
-        const cashFlowData = dataMap.get(this.lookbackDays);
-        const assetFlowData = dataMap.get(this.lookbackDays + 1);
-
-        if (!cashFlowData || !assetFlowData) {
-            logger.log('WARN', `[CashFlowDeployment] Missing dependency data for ${dateStr}. Skipping.`);
-            return null;
-        }
+        // --- REMOVED ---
+        // The check for cashFlowData and assetFlowData was here
+        // but has been moved up.
+        // --- END REMOVED ---
 
         const netSpendPct = cashFlowData.components?.trading_effect || 0;
         const netDepositPct = Math.abs(depositSignal.cash_flow_effect_proxy);
@@ -114,4 +122,4 @@ class CashFlowDeployment {
     reset() {}
 }
 
-module.exports = CashFlowDeployment;
+module.exports = CashFlowDeployment;

package/calculations/meta/cash-flow-liquidation.js

@@ -59,6 +59,18 @@ class CashFlowLiquidation {
             if (snap.exists) dataMap.set(idx, snap.data());
         });
 
+        // --- START MODIFICATION ---
+        // Check for THIS DATE's dependencies (cash-flow and asset-flow) FIRST.
+        const cashFlowData = dataMap.get(this.lookbackDays);
+        const assetFlowData = dataMap.get(this.lookbackDays + 1);
+
+        if (!cashFlowData || !assetFlowData) {
+            logger.log('WARN', `[CashFlowLiquidation] Missing critical dependency data (cash-flow or asset-flow) for ${dateStr}. Skipping to allow backfill.`);
+            return null; // This allows backfill
+        }
+        // --- END MODIFICATION ---
+
+
         // 2. Find the withdrawal signal
         let withdrawalSignal = null;
         let withdrawalSignalDay = null;
@@ -92,14 +104,10 @@ class CashFlowLiquidation {
             };
         }
 
-        // 3. Get today's data to find the correlation
-        const cashFlowData = dataMap.get(this.lookbackDays);
-        const assetFlowData = dataMap.get(this.lookbackDays + 1);
-
-        if (!cashFlowData || !assetFlowData) {
-            logger.log('WARN', `[CashFlowLiquidation] Missing dependency data for ${dateStr}. Skipping.`);
-            return null;
-        }
+        // --- REMOVED ---
+        // The check for cashFlowData and assetFlowData was here
+        // but has been moved up.
+        // --- END REMOVED ---
 
         // 'trading_effect' will be negative if the crowd is net-selling
         const netSellPct = cashFlowData.components?.trading_effect || 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aiden-shared-calculations-unified",
-  "version": "1.0.30",
+  "version": "1.0.32",
   "description": "Shared calculation modules for the BullTrackers Computation System.",
   "main": "index.js",
   "files": [

package/utils/price_data_provider.js

@@ -1,151 +1,99 @@
-// utils/price_data_provider.js
 const { Firestore } = require('@google-cloud/firestore');
 const firestore = new Firestore();
 
+// Config
 const PRICE_COLLECTION = 'asset_prices';
 const CACHE_DURATION_MS = 3600000; // 1 hour
 
-let cache = { timestamp: null, priceMap: null, meta: null };
+// Cache
+let cache = {
+  timestamp: null,
+  priceMap: null, // Will be { instrumentId: { "YYYY-MM-DD": price, ... } }
+};
+
+// In-progress fetch promise
 let fetchPromise = null;
 
-function isPlainObject(v) {
-  return v && typeof v === 'object' && !Array.isArray(v);
+/**
+ * Finds the most recent available price on or before a given date.
+ * @param {object} priceHistory - The map of { "YYYY-MM-DD": price }
+ * @param {string} dateStr - The target date string to start searching from.
+ * @param {number} [maxLookback=5] - Max days to look back (to skip weekends/holidays).
+ * @returns {number|null} The price, or null if not found.
+ */
+function _findPreviousAvailablePrice(priceHistory, dateStr, maxLookback = 5) {
+    if (!priceHistory) return null;
+
+    let checkDate = new Date(dateStr + 'T00:00:00Z');
+    
+    for (let i = 0; i < maxLookback; i++) {
+        const checkDateStr = checkDate.toISOString().slice(0, 10);
+        const price = priceHistory[checkDateStr];
+        
+        if (price !== undefined && price !== null && price > 0) {
+            return price; // Found it
+        }
+        
+        // If not found, look back one more day
+        checkDate.setUTCDate(checkDate.getUTCDate() - 1);
+    }
+    
+    // console.warn(`No price found for instrument within ${maxLookback} days of ${dateStr}`);
+    return null; // No price found within lookback
 }
 
 /**
- * Load all price data from 'asset_prices' collection.
- * This loader is resilient to two common shapes:
- * 1) Sharded docs: each doc contains many instrumentId keys:
- *    { "360": { prices: {...}, ticker: "NG.JUL23" }, "1028": { ... }, ... }
- * 2) Per-instrument docs: each doc id is an instrumentId and doc contains prices:
- *    doc.id = "360", doc.data() = { prices: {...}, ticker: "NG.JUL23", lastUpdated: ... }
- *
- * The returned priceMap will always be:
- * { "<instrumentId>": { prices: { "YYYY-MM-DD": number, ... }, ticker: string|null, lastUpdated: any|null } }
+ * Loads all sharded price data from the `asset_prices` collection.
+ * This is a heavy operation and should be cached.
  */
 async function loadAllPriceData() {
   const now = Date.now();
   if (cache.timestamp && (now - cache.timestamp < CACHE_DURATION_MS)) {
-    console.debug('[price_data_provider] returning cached priceMap', {
-      cachedAt: new Date(cache.timestamp).toISOString(),
-      instrumentCount: Object.keys(cache.priceMap || {}).length
-    });
     return cache.priceMap;
   }
 
-  if (fetchPromise) return fetchPromise;
+  if (fetchPromise) {
+    return fetchPromise;
+  }
 
   fetchPromise = (async () => {
-    console.log('[price_data_provider] fetching asset_prices collection from Firestore...');
+    console.log('Fetching and caching all asset price data...');
     const masterPriceMap = {};
-    let docCount = 0;
-    let shardDocs = 0;
-    let singleInstrumentDocs = 0;
+    
     try {
       const snapshot = await firestore.collection(PRICE_COLLECTION).get();
+      
       if (snapshot.empty) {
-        console.warn(`[price_data_provider] collection "${PRICE_COLLECTION}" is empty`);
-        return {};
+        throw new Error(`Price collection '${PRICE_COLLECTION}' is empty.`);
       }
 
+      // Loop through each shard document (e.g., "shard_0", "shard_1")
       snapshot.forEach(doc => {
-        docCount++;
-        const data = doc.data();
-
-        // Heuristic 1: if doc.data() has a 'prices' key and it's an object of date->number,
-        // treat this doc as a per-instrument document (doc.id = instrumentId).
-        if (isPlainObject(data) && 'prices' in data && isPlainObject(data.prices)) {
-          singleInstrumentDocs++;
-          const instrumentId = String(doc.id);
-          masterPriceMap[instrumentId] = {
-            prices: data.prices || {},
-            ticker: data.ticker || null,
-            lastUpdated: data.lastUpdated || null
-          };
-          return; // continue to next doc
-        }
-
-        // Heuristic 2: otherwise, check whether doc contains multiple instrumentId keys
-        // e.g., doc.data() looks like { "360": { prices: {...}, ticker: "NG.JUL23" }, "1028": { ... } }
-        const keys = Object.keys(data || {});
-        // detect shard-style by checking if at least one value is an object with .prices
-        const looksLikeShard = keys.length > 0 && keys.some(k => {
-          const v = data[k];
-          return isPlainObject(v) && ('prices' in v) && isPlainObject(v.prices);
-        });
-
-        if (looksLikeShard) {
-          shardDocs++;
-          for (const instrumentIdRaw of keys) {
-            try {
-              const entry = data[instrumentIdRaw];
-              if (!entry || !isPlainObject(entry)) continue;
-              if (!('prices' in entry) || !isPlainObject(entry.prices)) continue;
-
-              const instrumentId = String(instrumentIdRaw);
-              masterPriceMap[instrumentId] = {
-                prices: entry.prices || {},
-                ticker: entry.ticker || null,
-                lastUpdated: entry.lastUpdated || null
-              };
-            } catch (e) {
-              // resilience: skip malformed instrument entry
-              console.debug('[price_data_provider] skipping malformed entry in shard', {
-                docId: doc.id,
-                instrumentIdRaw,
-                err: e && e.message
-              });
-            }
+        const shardData = doc.data();
+        
+        // Loop through each instrumentId in the shard
+        for (const instrumentId in shardData) {
+          // Check if it's a valid instrument entry
+          if (shardData[instrumentId] && shardData[instrumentId].prices) {
+            masterPriceMap[instrumentId] = shardData[instrumentId].prices;
           }
-          return;
-        }
-
-        // If we get here, doc shape is unexpected. Log it for debugging and attempt
-        // a best-effort extraction: if doc keys look numeric and values are numbers
-        // or nested simple maps, we may try to treat doc.id as instrumentId fallback.
-        console.debug('[price_data_provider] unexpected doc shape in asset_prices', {
-          docId: doc.id,
-          sampleKeys: Object.keys(data || {}).slice(0, 5)
-        });
-
-        // Fallback: if doc has top-level numeric-like id and a single 'prices' like nested object, try doc.id
-        if (isPlainObject(data) && Object.keys(data).length > 0 && 'prices' in data) {
-          const instrumentId = String(doc.id);
-          masterPriceMap[instrumentId] = {
-            prices: data.prices || {},
-            ticker: data.ticker || null,
-            lastUpdated: data.lastUpdated || null
-          };
         }
       });
 
       cache = {
         timestamp: now,
         priceMap: masterPriceMap,
-        meta: {
-          instrumentCount: Object.keys(masterPriceMap).length,
-          docCount,
-          shardDocs,
-          singleInstrumentDocs
-        }
       };
-
-      console.log('[price_data_provider] load complete', cache.meta);
-
-      // also print a small sample to help debugging quickly
-      const sampleIds = Object.keys(masterPriceMap).slice(0, 10);
-      const samplePreview = sampleIds.map(id => ({
-        id,
-        ticker: masterPriceMap[id].ticker || null,
-        sampleDates: Object.keys(masterPriceMap[id].prices || {}).slice(0, 3)
-      }));
-      console.debug('[price_data_provider] sample preview:', samplePreview);
-
+      
+      console.log(`Successfully cached prices for ${Object.keys(masterPriceMap).length} instruments.`);
       return masterPriceMap;
+
     } catch (err) {
-      console.error('[price_data_provider] error loading price data:', err);
+      console.error('CRITICAL: Error loading price data:', err);
+      // On error, return an empty map but don't cache, so a future call can retry.
       return {};
     } finally {
+      // Clear the promise so the next call (if cache is stale) triggers a new fetch
       fetchPromise = null;
     }
   })();
@@ -154,40 +102,41 @@ async function loadAllPriceData() {
 }
 
 /**
- * Get daily price change percentage. Accepts priceMap with shape produced by loadAllPriceData().
- * Returns number (e.g., 0.1 for +10%) or null if not available.
+ * A helper to safely get the price change percentage between two dates.
+ * --- THIS IS THE MODIFIED FUNCTION ---
+ * @param {string} instrumentId - The instrument ID.
+ * @param {string} yesterdayStr - YYYY-MM-DD date string for yesterday.
+ * @param {string} todayStr - YYYY-MM-DD date string for today.
+ * @param {object} priceMap - The master price map from loadAllPriceData().
+ * @returns {number|null} The percentage change (e.g., 0.10 for +10%), or null if data is missing.
  */
 function getDailyPriceChange(instrumentId, yesterdayStr, todayStr, priceMap) {
-  if (!priceMap) return null;
-  const id = String(instrumentId);
-  const entry = priceMap[id];
-
-  if (!entry) return null;
-
-  // entry may be { prices: {...}, ticker, lastUpdated } OR legacy style where entry is the raw prices map
-  const prices = (isPlainObject(entry) && 'prices' in entry) ? entry.prices : entry;
-
-  if (!isPlainObject(prices)) return null;
-
-  const p1 = prices[yesterdayStr];
-  const p2 = prices[todayStr];
-
-  // log missing specifics for traceability (not too spammy)
-  if (p1 == null || p2 == null) {
-    // caller should already log, but this helps track date mismatches
-    console.debug('[price_data_provider] missing date price', { instrumentId: id, yesterdayStr, todayStr, hasP1: p1 != null, hasP2: p2 != null });
-    return null;
-  }
-
-  if (typeof p1 !== 'number' || typeof p2 !== 'number' || p1 <= 0) {
-    console.debug('[price_data_provider] invalid price values', { instrumentId: id, p1, p2 });
-    return null;
-  }
-
-  return (p2 - p1) / p1;
+    if (!priceMap || !priceMap[instrumentId]) {
+        return null; // No price data for this instrument
+    }
+    
+    const instrumentPrices = priceMap[instrumentId];
+
+    // Find the most recent available price *on or before* the target dates
+    const priceDay1 = _findPreviousAvailablePrice(instrumentPrices, yesterdayStr, 5);
+    const priceDay2 = _findPreviousAvailablePrice(instrumentPrices, todayStr, 5);
+
+    if (priceDay1 && priceDay2) {
+        // We found prices, now check if they are the *same* price
+        // (e.g., if today is Sunday, priceDay2 would be Friday's price.
+        // If yesterday was Saturday, priceDay1 would *also* be Friday's price).
+        if (priceDay1 === priceDay2) {
+            return 0.0; // No change between the two most recent *available* dates
+        }
+        return (priceDay2 - priceDay1) / priceDay1;
+    }
+    
+    return null; // Missing one or both dates, even after lookback
 }
 
+
 module.exports = {
-  loadAllPriceData,
-  getDailyPriceChange
+    loadAllPriceData,
+    getDailyPriceChange
+    // Add the helper to exports if you want, but it's not required
 };