aiden-shared-calculations-unified 1.0.28 → 1.0.30

package/calculations/behavioural/historical/asset_crowd_flow.js

@@ -81,64 +81,85 @@ class AssetCrowdFlow {
 
     async getResult() {
         if (this.user_count === 0 || !this.dates.today) {
-            return {}; // No users processed or dates not found
+            console.warn('[AssetCrowdFlow] No users processed or dates missing.');
+            return {};
         }
 
-        // Load dependencies (prices and mappings) in parallel
+        // Load priceMap and mappings if not loaded
         if (!this.priceMap || !this.mappings) {
-            const [priceData, mappingData] = await Promise.all([
-                loadAllPriceData(),
-                loadInstrumentMappings()
-            ]);
-            this.priceMap = priceData;
-            this.mappings = mappingData;
+            try {
+                const [priceData, mappingData] = await Promise.all([
+                    loadAllPriceData(),
+                    loadInstrumentMappings()
+                ]);
+                this.priceMap = priceData;
+                this.mappings = mappingData;
+            } catch (err) {
+                console.error('[AssetCrowdFlow] Failed to load dependencies:', err);
+                return {};
+            }
         }
-        
+
         const finalResults = {};
         const todayStr = this.dates.today;
         const yesterdayStr = this.dates.yesterday;
 
-        for (const instrumentId in this.asset_values) {
-            const ticker = this.mappings.instrumentToTicker[instrumentId] || `id_${instrumentId}`;
-            
-            // 1. Calculate average % values
-            const avg_day1_value = this.asset_values[instrumentId].day1_value_sum / this.user_count;
-            const avg_day2_value = this.asset_values[instrumentId].day2_value_sum / this.user_count;
-            
-            // 2. Get the actual price change
-            const priceChangePct = getDailyPriceChange(instrumentId, yesterdayStr, todayStr, this.priceMap);
+        for (const rawInstrumentId in this.asset_values) {
+            const instrumentId = String(rawInstrumentId); // normalize
+            const ticker = this.mappings.instrumentToTicker?.[instrumentId] || `id_${instrumentId}`;
+
+            const avg_day1_value = this.asset_values[rawInstrumentId].day1_value_sum / this.user_count;
+            const avg_day2_value = this.asset_values[rawInstrumentId].day2_value_sum / this.user_count;
+
+            let priceChangePct = null;
+
+            // Check priceMap presence
+            if (!this.priceMap || !this.priceMap[instrumentId]) {
+                console.debug(`[AssetCrowdFlow] Missing priceMap entry for instrumentId ${instrumentId} (${ticker})`);
+            } else {
+                const priceDay1 = this.priceMap[instrumentId][yesterdayStr];
+                const priceDay2 = this.priceMap[instrumentId][todayStr];
+
+                if (priceDay1 == null) console.debug(`[AssetCrowdFlow] Missing price for ${instrumentId} (${ticker}) on ${yesterdayStr}`);
+                if (priceDay2 == null) console.debug(`[AssetCrowdFlow] Missing price for ${instrumentId} (${ticker}) on ${todayStr}`);
+
+                if (priceDay1 != null && priceDay2 != null && priceDay1 > 0) {
+                    priceChangePct = (priceDay2 - priceDay1) / priceDay1;
+                }
+            }
 
             if (priceChangePct === null) {
-                // Cannot calculate if price data is missing for either day
                 finalResults[ticker] = {
                     net_crowd_flow_pct: 0,
-                    error: "Missing price data for calculation."
+                    avg_value_day1: avg_day1_value,
+                    avg_value_day2: avg_day2_value,
+                    price_change_pct: null,
+                    user_sample_size: this.user_count,
+                    warning: 'Missing price data for calculation'
                 };
                 continue;
             }
 
-            // 3. Calculate the expected value (the "price-move" effect)
-            // We use avg_day1_value as the base. The cash flow proxy calculation
-            // uses (avg_value - avg_invested) because it's solving for a different unknown.
-            // Here, we are solving for flow *relative to the asset itself*.
+            // Calculate expected day2 value from price movement
             const expected_day2_value = avg_day1_value * (1 + priceChangePct);
 
-            // 4. Find the signal (the "crowd-flow" effect)
+            // Net crowd flow = actual minus expected
             const net_crowd_flow_pct = avg_day2_value - expected_day2_value;
 
             finalResults[ticker] = {
-                net_crowd_flow_pct: net_crowd_flow_pct,
-                avg_value_day1_pct: avg_day1_value,
-                avg_value_day2_pct: avg_day2_value,
-                expected_value_day2_pct: expected_day2_value,
+                net_crowd_flow_pct,
+                avg_value_day1,
+                avg_value_day2,
+                expected_day2_value,
                 price_change_pct: priceChangePct,
                 user_sample_size: this.user_count
             };
         }
-        
+
         return finalResults;
     }
 
+
     reset() {
         this.asset_values = {};
         this.user_count = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aiden-shared-calculations-unified",
-  "version": "1.0.28",
+  "version": "1.0.30",
   "description": "Shared calculation modules for the BullTrackers Computation System.",
   "main": "index.js",
   "files": [

package/utils/price_data_provider.js

@@ -1,71 +1,151 @@
+// utils/price_data_provider.js
 const { Firestore } = require('@google-cloud/firestore');
 const firestore = new Firestore();
 
-// Config
 const PRICE_COLLECTION = 'asset_prices';
 const CACHE_DURATION_MS = 3600000; // 1 hour
 
-// Cache
-let cache = {
-  timestamp: null,
-  priceMap: null, // Will be { instrumentId: { "YYYY-MM-DD": price, ... } }
-};
-
-// In-progress fetch promise
+let cache = { timestamp: null, priceMap: null, meta: null };
 let fetchPromise = null;
 
+function isPlainObject(v) {
+  return v && typeof v === 'object' && !Array.isArray(v);
+}
+
 /**
- * Loads all sharded price data from the `asset_prices` collection.
- * This is a heavy operation and should be cached.
+ * Load all price data from 'asset_prices' collection.
+ * This loader is resilient to two common shapes:
+ * 1) Sharded docs: each doc contains many instrumentId keys:
+ *    { "360": { prices: {...}, ticker: "NG.JUL23" }, "1028": { ... }, ... }
+ * 2) Per-instrument docs: each doc id is an instrumentId and doc contains prices:
+ *    doc.id = "360", doc.data() = { prices: {...}, ticker: "NG.JUL23", lastUpdated: ... }
+ *
+ * The returned priceMap will always be:
+ * { "<instrumentId>": { prices: { "YYYY-MM-DD": number, ... }, ticker: string|null, lastUpdated: any|null } }
  */
 async function loadAllPriceData() {
   const now = Date.now();
   if (cache.timestamp && (now - cache.timestamp < CACHE_DURATION_MS)) {
+    console.debug('[price_data_provider] returning cached priceMap', {
+      cachedAt: new Date(cache.timestamp).toISOString(),
+      instrumentCount: Object.keys(cache.priceMap || {}).length
+    });
     return cache.priceMap;
   }
 
-  if (fetchPromise) {
-    return fetchPromise;
-  }
+  if (fetchPromise) return fetchPromise;
 
   fetchPromise = (async () => {
-    console.log('Fetching and caching all asset price data...');
+    console.log('[price_data_provider] fetching asset_prices collection from Firestore...');
     const masterPriceMap = {};
-    
+    let docCount = 0;
+    let shardDocs = 0;
+    let singleInstrumentDocs = 0;
     try {
       const snapshot = await firestore.collection(PRICE_COLLECTION).get();
-      
       if (snapshot.empty) {
-        throw new Error(`Price collection '${PRICE_COLLECTION}' is empty.`);
+        console.warn(`[price_data_provider] collection "${PRICE_COLLECTION}" is empty`);
+        return {};
       }
 
-      // Loop through each shard document (e.g., "shard_0", "shard_1")
       snapshot.forEach(doc => {
-        const shardData = doc.data();
-        
-        // Loop through each instrumentId in the shard
-        for (const instrumentId in shardData) {
-          // Check if it's a valid instrument entry
-          if (shardData[instrumentId] && shardData[instrumentId].prices) {
-            masterPriceMap[instrumentId] = shardData[instrumentId].prices;
+        docCount++;
+        const data = doc.data();
+
+        // Heuristic 1: if doc.data() has a 'prices' key and it's an object of date->number,
+        // treat this doc as a per-instrument document (doc.id = instrumentId).
+        if (isPlainObject(data) && 'prices' in data && isPlainObject(data.prices)) {
+          singleInstrumentDocs++;
+          const instrumentId = String(doc.id);
+          masterPriceMap[instrumentId] = {
+            prices: data.prices || {},
+            ticker: data.ticker || null,
+            lastUpdated: data.lastUpdated || null
+          };
+          return; // continue to next doc
+        }
+
+        // Heuristic 2: otherwise, check whether doc contains multiple instrumentId keys
+        // e.g., doc.data() looks like { "360": { prices: {...}, ticker: "NG.JUL23" }, "1028": { ... } }
+        const keys = Object.keys(data || {});
+        // detect shard-style by checking if at least one value is an object with .prices
+        const looksLikeShard = keys.length > 0 && keys.some(k => {
+          const v = data[k];
+          return isPlainObject(v) && ('prices' in v) && isPlainObject(v.prices);
+        });
+
+        if (looksLikeShard) {
+          shardDocs++;
+          for (const instrumentIdRaw of keys) {
+            try {
+              const entry = data[instrumentIdRaw];
+              if (!entry || !isPlainObject(entry)) continue;
+              if (!('prices' in entry) || !isPlainObject(entry.prices)) continue;
+
+              const instrumentId = String(instrumentIdRaw);
+              masterPriceMap[instrumentId] = {
+                prices: entry.prices || {},
+                ticker: entry.ticker || null,
+                lastUpdated: entry.lastUpdated || null
+              };
+            } catch (e) {
+              // resilience: skip malformed instrument entry
+              console.debug('[price_data_provider] skipping malformed entry in shard', {
+                docId: doc.id,
+                instrumentIdRaw,
+                err: e && e.message
+              });
+            }
           }
+          return;
+        }
+
+        // If we get here, doc shape is unexpected. Log it for debugging and attempt
+        // a best-effort extraction: if doc keys look numeric and values are numbers
+        // or nested simple maps, we may try to treat doc.id as instrumentId fallback.
+        console.debug('[price_data_provider] unexpected doc shape in asset_prices', {
+          docId: doc.id,
+          sampleKeys: Object.keys(data || {}).slice(0, 5)
+        });
+
+        // Fallback: if doc has top-level numeric-like id and a single 'prices' like nested object, try doc.id
+        if (isPlainObject(data) && Object.keys(data).length > 0 && 'prices' in data) {
+          const instrumentId = String(doc.id);
+          masterPriceMap[instrumentId] = {
+            prices: data.prices || {},
+            ticker: data.ticker || null,
+            lastUpdated: data.lastUpdated || null
+          };
         }
       });
 
       cache = {
         timestamp: now,
         priceMap: masterPriceMap,
+        meta: {
+          instrumentCount: Object.keys(masterPriceMap).length,
+          docCount,
+          shardDocs,
+          singleInstrumentDocs
+        }
       };
-      
-      console.log(`Successfully cached prices for ${Object.keys(masterPriceMap).length} instruments.`);
-      return masterPriceMap;
 
+      console.log('[price_data_provider] load complete', cache.meta);
+
+      // also print a small sample to help debugging quickly
+      const sampleIds = Object.keys(masterPriceMap).slice(0, 10);
+      const samplePreview = sampleIds.map(id => ({
+        id,
+        ticker: masterPriceMap[id].ticker || null,
+        sampleDates: Object.keys(masterPriceMap[id].prices || {}).slice(0, 3)
+      }));
+      console.debug('[price_data_provider] sample preview:', samplePreview);
+
+      return masterPriceMap;
     } catch (err) {
-      console.error('CRITICAL: Error loading price data:', err);
-      // On error, return an empty map but don't cache, so a future call can retry.
+      console.error('[price_data_provider] error loading price data:', err);
       return {};
     } finally {
-      // Clear the promise so the next call (if cache is stale) triggers a new fetch
       fetchPromise = null;
     }
   })();
@@ -74,30 +154,40 @@ async function loadAllPriceData() {
 }
 
 /**
- * A helper to safely get the price change percentage between two dates.
- * @param {string} instrumentId - The instrument ID.
- * @param {string} yesterdayStr - YYYY-MM-DD date string for yesterday.
- * @param {string} todayStr - YYYY-MM-DD date string for today.
- * @param {object} priceMap - The master price map from loadAllPriceData().
- * @returns {number|null} The percentage change (e.g., 0.10 for +10%), or null if data is missing.
+ * Get daily price change percentage. Accepts priceMap with shape produced by loadAllPriceData().
+ * Returns number (e.g., 0.1 for +10%) or null if not available.
  */
 function getDailyPriceChange(instrumentId, yesterdayStr, todayStr, priceMap) {
-    if (!priceMap || !priceMap[instrumentId]) {
-        return null; // No price data for this instrument
-    }
-    
-    const priceDay1 = priceMap[instrumentId][yesterdayStr];
-    const priceDay2 = priceMap[instrumentId][todayStr];
+  if (!priceMap) return null;
+  const id = String(instrumentId);
+  const entry = priceMap[id];
 
-    if (priceDay1 && priceDay2 && priceDay1 > 0) {
-        return (priceDay2 - priceDay1) / priceDay1;
-    }
-    
-    return null; // Missing one or both dates, or division by zero
-}
+  if (!entry) return null;
+
+  // entry may be { prices: {...}, ticker, lastUpdated } OR legacy style where entry is the raw prices map
+  const prices = (isPlainObject(entry) && 'prices' in entry) ? entry.prices : entry;
 
+  if (!isPlainObject(prices)) return null;
+
+  const p1 = prices[yesterdayStr];
+  const p2 = prices[todayStr];
+
+  // log missing specifics for traceability (not too spammy)
+  if (p1 == null || p2 == null) {
+    // caller should already log, but this helps track date mismatches
+    console.debug('[price_data_provider] missing date price', { instrumentId: id, yesterdayStr, todayStr, hasP1: p1 != null, hasP2: p2 != null });
+    return null;
+  }
+
+  if (typeof p1 !== 'number' || typeof p2 !== 'number' || p1 <= 0) {
+    console.debug('[price_data_provider] invalid price values', { instrumentId: id, p1, p2 });
+    return null;
+  }
+
+  return (p2 - p1) / p1;
+}
 
 module.exports = {
-    loadAllPriceData,
-    getDailyPriceChange
+  loadAllPriceData,
+  getDailyPriceChange
 };