bulltrackers-module 1.0.249 → 1.0.250

package/functions/root-data-indexer/index.js

@@ -0,0 +1,168 @@
+/**
+ * @fileoverview Root Data Indexer
+ * Runs daily to index exactly what data is available for every date.
+ * Creates a "Source of Truth" map in Firestore for the Dispatcher/Computation System.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit');
+
+// Hardcoded verification blocks as per logic requirements
+const CANARY_BLOCK_ID = '19M';
+const CANARY_PART_ID = 'part_0';
+const PRICE_SHARD_ID = 'shard_0';
+
+/**
+ * Main pipe: pipe.maintenance.runRootDataIndexer
+ */
+exports.runRootDataIndexer = async (config, dependencies) => {
+    const { db, logger } = dependencies;
+    const { 
+        availabilityCollection, 
+        earliestDate,
+        collections 
+    } = config;
+
+    logger.log('INFO', '[RootDataIndexer] Starting Root Data Availability Scan...');
+
+    // 1. Pre-fetch Price Data Availability (Optimization)
+    // Instead of reading the shard N times, we read it once and map available dates.
+    const priceAvailabilitySet = new Set();
+    try {
+        const priceShardRef = db.collection(collections.prices).doc(PRICE_SHARD_ID);
+        const priceSnap = await priceShardRef.get();
+        if (priceSnap.exists) {
+            const data = priceSnap.data();
+            // Iterate over all instruments in this shard to find any available dates
+            Object.values(data).forEach(instrument => {
+                if (instrument.prices) {
+                    Object.keys(instrument.prices).forEach(dateKey => {
+                        // Validate format YYYY-MM-DD
+                        if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) {
+                            priceAvailabilitySet.add(dateKey);
+                        }
+                    });
+                }
+            });
+        }
+        logger.log('INFO', `[RootDataIndexer] Loaded price availability map. Found prices for ${priceAvailabilitySet.size} unique dates.`);
+    } catch (e) {
+        logger.log('ERROR', `[RootDataIndexer] Failed to load price shard ${PRICE_SHARD_ID}. Price availability will be false.`, { error: e.message });
+    }
+
+    // 2. Determine Date Range (Earliest -> Tomorrow)
+    const start = new Date(earliestDate || '2023-01-01');
+    const end = new Date();
+    end.setDate(end.getDate() + 1); // Look ahead 1 day
+
+    const datesToScan = [];
+    for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
+        datesToScan.push(d.toISOString().slice(0, 10));
+    }
+
+    logger.log('INFO', `[RootDataIndexer] Scanning ${datesToScan.length} dates from ${datesToScan[0]} to ${datesToScan[datesToScan.length-1]}`);
+
+    // 3. Scan in Parallel
+    const limit = pLimit(20); // Concurrent date checks
+    let updatesCount = 0;
+
+    const promises = datesToScan.map(dateStr => limit(async () => {
+        try {
+            const availability = {
+                date: dateStr,
+                lastUpdated: FieldValue.serverTimestamp(),
+                
+                // Defaults
+                hasPortfolio: false,
+                hasHistory: false,
+                hasSocial: false,
+                hasInsights: false,
+                hasPrices: false,
+                
+                // Detailed breakdown
+                details: {
+                    normalPortfolio: false,
+                    speculatorPortfolio: false,
+                    normalHistory: false,
+                    speculatorHistory: false
+                }
+            };
+
+            // A. User Portfolios (Normal) - /NormalUserPortfolios/19M/snapshots/{date}/parts/part_0
+            const normPortRef = db.collection(collections.normalPortfolios)
+                .doc(CANARY_BLOCK_ID)
+                .collection('snapshots')
+                .doc(dateStr)
+                .collection('parts')
+                .doc(CANARY_PART_ID);
+            
+            // B. Speculator Portfolios - /SpeculatorPortfolios/19M/snapshots/{date}/parts/part_0
+            const specPortRef = db.collection(collections.speculatorPortfolios)
+                .doc(CANARY_BLOCK_ID)
+                .collection('snapshots')
+                .doc(dateStr)
+                .collection('parts')
+                .doc(CANARY_PART_ID);
+
+            // C. User Trade History - /NormalUserTradeHistory/19M/snapshots/{date}/parts/part_0
+            const normHistRef = db.collection(collections.normalHistory)
+                .doc(CANARY_BLOCK_ID)
+                .collection('snapshots')
+                .doc(dateStr)
+                .collection('parts')
+                .doc(CANARY_PART_ID);
+
+            // D. Speculator Trade History - /SpeculatorTradeHistory/19M/snapshots/{date}/parts/part_0
+            const specHistRef = db.collection(collections.speculatorHistory)
+                .doc(CANARY_BLOCK_ID)
+                .collection('snapshots')
+                .doc(dateStr)
+                .collection('parts')
+                .doc(CANARY_PART_ID);
+
+            // E. Insights Data - /daily_instrument_insights/{date}
+            const insightsRef = db.collection(collections.insights).doc(dateStr);
+
+            // F. Social Data - /daily_social_insights/{date}/posts (Check for ANY doc)
+            const socialPostsRef = db.collection(collections.social).doc(dateStr).collection('posts');
+
+            // --- EXECUTE CHECKS ---
+            const [
+                normPortSnap, specPortSnap,
+                normHistSnap, specHistSnap,
+                insightsSnap, socialQuerySnap
+            ] = await Promise.all([
+                normPortRef.get(), specPortRef.get(),
+                normHistRef.get(), specHistRef.get(),
+                insightsRef.get(), 
+                socialPostsRef.limit(1).get()
+            ]);
+
+            // Evaluate Findings
+            availability.details.normalPortfolio = normPortSnap.exists;
+            availability.details.speculatorPortfolio = specPortSnap.exists;
+            availability.hasPortfolio = normPortSnap.exists || specPortSnap.exists;
+
+            availability.details.normalHistory = normHistSnap.exists;
+            availability.details.speculatorHistory = specHistSnap.exists;
+            availability.hasHistory = normHistSnap.exists || specHistSnap.exists;
+
+            availability.hasInsights = insightsSnap.exists;
+            availability.hasSocial = !socialQuerySnap.empty;
+
+            // G. Prices (From pre-loaded set)
+            availability.hasPrices = priceAvailabilitySet.has(dateStr);
+
+            // Write to Index
+            await db.collection(availabilityCollection).doc(dateStr).set(availability);
+            updatesCount++;
+
+        } catch (e) {
+            logger.log('ERROR', `[RootDataIndexer] Failed to index ${dateStr}`, { error: e.message });
+        }
+    }));
+
+    await Promise.all(promises);
+    logger.log('SUCCESS', `[RootDataIndexer] Indexing complete. Updated ${updatesCount} dates.`);
+    return { success: true, count: updatesCount };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.249",
+  "version": "1.0.250",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -18,7 +18,8 @@
     "functions/appscript-api/",
     "functions/social-orchestrator/",
     "functions/social-task-handler/",
-    "functions/price-backfill/"
+    "functions/price-backfill/",
+    "functions/root-data-indexer/"
   ],
   "keywords": [
     "bulltrackers",