bulltrackers-module 1.0.721 → 1.0.723

package/functions/root-data-indexer/index.js

@@ -1,563 +1,256 @@
 /**
  * @fileoverview Root Data Indexer
  * Runs daily to index exactly what data is available for every date.
- * UPDATED: Includes 'targetDate' optimization for fast single-day updates.
- * UPDATED: Includes explicit checks for Signed-In User Portfolios, History, AND Social Data.
+ * REFACTORED: Simplifies checks by relying on BigQuery for migrated data types.
+ * RETAINS: Firestore checks for Verifications, Normal/Speculator users, and Generic Social.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit         = require('p-limit');
 
 const CANARY_BLOCK_ID = '19M';
-const PRICE_SHARD_ID  = 'shard_0';
 
 /**
  * Helper function to check if any part document exists in a parts collection
- * @param {Firestore.CollectionReference} partsCollectionRef - Reference to the parts collection
- * @returns {Promise<boolean>} - True if any part_* document exists
+ * Used only for legacy Retail (Normal/Speculator) data.
  */
 async function checkAnyPartExists(partsCollectionRef) {
     try {
-        // List all documents in the parts collection
-        const snapshot = await partsCollectionRef.limit(10).get();
-        if (snapshot.empty) return false;
-        
-        // Check if any document ID starts with 'part_'
-        for (const doc of snapshot.docs) {
-            if (doc.id.startsWith('part_')) {
-                return true;
-            }
-        }
-        return false;
+        const snapshot = await partsCollectionRef.limit(1).get();
+        return !snapshot.empty;
     } catch (error) {
         return false;
     }
 }
 
-/**
- * Helper function to check if any shard document exists in a collection
- * @param {Firestore.CollectionReference} collectionRef - Reference to the collection
- * @returns {Promise<boolean>} - True if any shard_* document exists
- */
-async function checkAnyShardExists(collectionRef) {
-    try {
-        // List all documents in the collection
-        const snapshot = await collectionRef.limit(10).get();
-        if (snapshot.empty) return false;
-        
-        // Check if any document ID starts with 'shard_'
-        for (const doc of snapshot.docs) {
-            if (doc.id.startsWith('shard_')) {
-                return true;
-            }
-        }
-        return false;
-    } catch (error) {
-        return false;
-    }
-} 
-
 exports.runRootDataIndexer = async (config, dependencies) => {
     const { db, logger } = dependencies;
     const { 
         availabilityCollection, 
         earliestDate,
         collections = {},
-        targetDate // [NEW] Optional parameter to scan a single specific date
+        targetDate
     } = config;
 
-    const PRICE_COLLECTION_NAME = 'asset_prices';
-    
-    // Collection Names (Fail-safe defaults)
-    const PI_SOCIAL_COLL_NAME      = collections.piSocial || 'pi_social_posts';
-    const SIGNED_IN_SOCIAL_COLL_NAME = collections.signedInUserSocialCollection || 'signed_in_users_social';
-    
-    // Ensure all required collections have defaults to prevent "collectionPath is not valid" errors
+    // Collection Config (Retained for Firestore-based types)
     const safeCollections = {
         normalPortfolios: collections.normalPortfolios || 'NormalUserPortfolios',
         speculatorPortfolios: collections.speculatorPortfolios || 'SpeculatorPortfolios',
         normalHistory: collections.normalHistory || 'NormalUserTradeHistory',
         speculatorHistory: collections.speculatorHistory || 'SpeculatorTradeHistory',
-        insights: collections.insights || 'daily_instrument_insights',
-        social: collections.social || 'daily_social_insights',
-        prices: collections.prices || PRICE_COLLECTION_NAME,
-        piRankings: collections.piRankings || 'popular_investor_rankings',
-        piPortfolios: collections.piPortfolios || 'pi_portfolios_overall',
-        piDeepPortfolios: collections.piDeepPortfolios || 'pi_portfolios_deep',
-        piHistory: collections.piHistory || 'pi_trade_history',
-        signedInUsers: collections.signedInUsers || 'signed_in_users',
-        signedInHistory: collections.signedInHistory || 'signed_in_user_history',
+        social: collections.social || 'daily_social_insights', // Generic Social
         verifications: collections.verifications || 'user_verifications',
-        // New Root Data Types for Profile Metrics
-        piRatings: collections.piRatings || 'PIRatingsData',
-        piPageViews: collections.piPageViews || 'PIPageViewsData',
-        watchlistMembership: collections.watchlistMembership || 'WatchlistMembershipData',
-        piAlertHistory: collections.piAlertHistory || 'PIAlertHistoryData',
-        piMasterList: collections.piMasterList || 'system_state', // [NEW] Collection for master list
-        ...collections // Allow overrides
+        ...collections
     };
 
     const scanMode = targetDate ? 'SINGLE_DATE' : 'FULL_SCAN';
-    logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
-    
-    // 1. Price Availability - Read from date tracking documents
-    // Find the latest price tracking document and extract available dates
-    const priceAvailabilitySet = new Set();
+    logger.log('INFO', `[RootDataIndexer] Starting Scan... Mode: ${scanMode}`, { targetDate });
     
-    // Get price tracking collection name from registry if available
-    let priceTrackingCollectionName = 'pricedatastoreddates';
-    if (dependencies.collectionRegistry && dependencies.collectionRegistry.getCollectionPath) {
-        try {
-            const trackingPath = dependencies.collectionRegistry.getCollectionPath('rootData', 'priceTracking', { fetchDate: '2025-01-01' });
-            priceTrackingCollectionName = trackingPath.split('/')[0];
-        } catch (e) {
-            logger.log('WARN', `[RootDataIndexer] Failed to get price tracking collection from registry, using default: ${e.message}`);
-        }
-    }
-    
-    try {
-        // Get the latest price date tracking document
-        const dateTrackingRef = db.collection(priceTrackingCollectionName)
-            .orderBy('fetchDate', 'desc')
-            .limit(1);
-        
-        const dateTrackingSnapshot = await dateTrackingRef.get();
-        
-        if (!dateTrackingSnapshot.empty) {
-            const latestTrackingDoc = dateTrackingSnapshot.docs[0].data();
-            const datesAvailable = latestTrackingDoc.datesAvailable || [];
-            const fetchDate = latestTrackingDoc.fetchDate;
-            
-            // Add all dates from the tracking document
-            datesAvailable.forEach(dateKey => {
-                if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) {
-                    priceAvailabilitySet.add(dateKey);
-                }
-            });
-            
-            // IMPORTANT: If the tracking document was written for today (fetchDate matches targetDate),
-            // we should consider prices available for that date even if the API didn't return that exact date.
-            // This is because the price fetcher ran for that date and stored prices (even if they're historical).
-            if (targetDate && fetchDate === targetDate) {
-                priceAvailabilitySet.add(targetDate);
-                logger.log('INFO', `[RootDataIndexer] Added fetchDate (${fetchDate}) to price availability set for target date check`);
-            }
-            
-            logger.log('INFO', `[RootDataIndexer] Loaded ${priceAvailabilitySet.size} price dates from tracking document (fetchDate: ${fetchDate})`);
-            
-            // Debug: Log a sample of dates and check if target date is present
-            if (targetDate) {
-                const sampleDates = Array.from(priceAvailabilitySet).slice(0, 5);
-                const hasTargetDate = priceAvailabilitySet.has(targetDate);
-                logger.log('INFO', `[RootDataIndexer] Price availability check for ${targetDate}: ${hasTargetDate ? 'FOUND' : 'NOT FOUND'}. Sample dates: ${sampleDates.join(', ')}`);
-            }
-        } else {
-            logger.log('WARN', '[RootDataIndexer] No price date tracking documents found. Falling back to empty set.');
-        }
-    } catch (e) {
-        logger.log('ERROR', '[RootDataIndexer] Failed to load price date tracking document.', { error: e.message });
-        // Fallback: try to sample shards if tracking document fails
-        if (!targetDate) {
-            try {
-                const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
-                const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
-                
-                if (!priceShardsSnapshot.empty) {
-                    for (const shardDoc of priceShardsSnapshot.docs) {
-                        if (shardDoc.id.startsWith('shard_')) {
-                            const data = shardDoc.data();
-                            Object.values(data).forEach(instrument => { 
-                                if (instrument && instrument.prices) { 
-                                    Object.keys(instrument.prices).forEach(dateKey => { 
-                                        if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
-                                            priceAvailabilitySet.add(dateKey); 
-                                        } 
-                                    }); 
-                                } 
-                            });
-                        }
-                    }
-                    logger.log('INFO', `[RootDataIndexer] Fallback: Loaded ${priceAvailabilitySet.size} price dates from shard sampling.`);
-                }
-            } catch (fallbackError) {
-                logger.log('ERROR', '[RootDataIndexer] Fallback shard sampling also failed.', { error: fallbackError.message });
-            }
-        }
-    }
-
-    // 2. Determine Date Range
+    // 1. Determine Date Range
     const datesToScan = [];
     if (targetDate) {
-        // [NEW] Single Date Optimization
         datesToScan.push(targetDate);
     } else {
-        // [OLD] Full History
         const start = new Date(earliestDate || '2023-01-01');
         const end   = new Date();
         end.setDate(end.getDate() + 1); 
-
         for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) { 
             datesToScan.push(d.toISOString().slice(0, 10)); 
         }
     }
 
-    // 3. Scan in Parallel
     const limit = pLimit(20); 
     let updatesCount = 0;
 
     const promises = datesToScan.map(dateStr => limit(async () => {
         try {
-            // Define Time Range for Social Query (Full Day UTC)
-            // Use UTC methods to ensure correct timezone handling
-            const dayStart = new Date(dateStr + 'T00:00:00.000Z');
-            const dayEnd = new Date(dateStr + 'T23:59:59.999Z');
-
             const availability = {
                 date: dateStr,
                 lastUpdated: FieldValue.serverTimestamp(),
+                // High-level aggregates
                 hasPortfolio: false,
                 hasHistory:   false,
                 hasSocial:    false,
                 hasInsights:  false,
                 hasPrices:    false,
                 details: {
+                    // --- FIRESTORE DRIVEN (Not Migrated) ---
                     normalPortfolio:     false,
                     speculatorPortfolio: false,
                     normalHistory:       false,
                     speculatorHistory:   false,
+                    signedInUserVerification: false, // Explicitly Firestore
+                    
+                    // --- BIGQUERY DRIVEN (Migrated) ---
                     piRankings:          false, 
                     piPortfolios:        false, 
-                    piDeepPortfolios:    false,
+                    piDeepPortfolios:    false, // Deprecated/Merged into piPortfolios
                     piHistory:           false, 
-                    piSocial:            false, // Specific Flag for PI
+                    piSocial:            false, 
                     signedInUserPortfolio: false,
                     signedInUserHistory:   false, 
-                    signedInUserVerification: false,
-                    signedInSocial:        false,  // Specific Flag for Signed-In
-                    // New Root Data Types for Profile Metrics
+                    signedInSocial:        false,
+                    
+                    // Profile Metrics (BigQuery)
                     piRatings:             false,
                     piPageViews:           false,
                     watchlistMembership:   false,
                     piAlertHistory:        false,
-                    piMasterList:          false // [NEW] Flag for Master List
+                    piMasterList:          false
                 }
             };
 
-            // --- Define Refs & Check Paths ---
-            
             // =========================================================================
-            // BIGQUERY FIRST: Check BigQuery tables for data availability
+            // 1. BIGQUERY CHECKS (Primary for most data)
             // =========================================================================
-            let bigqueryHasPortfolio = false;
-            let bigqueryHasHistory = false;
-            let bigqueryHasSocial = false;
-            let bigqueryHasInsights = false;
-            let bigqueryHasPrices = false;
-            let bigqueryHasPIRankings = false;
-            let bigqueryHasPIMasterList = false;
-            
+            let bqData = {
+                portfolio: {}, history: {}, social: {}, 
+                insights: [], prices: {}, rankings: [], masterList: {},
+                ratings: {}, pageViews: {}, watchlists: {}, alerts: {}
+            };
+
             if (process.env.BIGQUERY_ENABLED !== 'false') {
                 try {
                     const { 
-                        queryPortfolioData, 
-                        queryHistoryData, 
-                        querySocialData,
-                        queryInstrumentInsights,
-                        queryAssetPrices,
-                        queryPIRankings,
-                        queryPIMasterList
+                        queryPortfolioData, queryHistoryData, querySocialData,
+                        queryInstrumentInsights, queryAssetPrices, queryPIRankings,
+                        queryPIMasterList, queryPIRatings, queryPIPageViews, 
+                        queryWatchlistMembership, queryPIAlertHistory
                     } = require('../core/utils/bigquery_utils');
                     
-                    // Check BigQuery for today's data (or target date)
-                    const [portfolioData, historyData, socialData, insightsData, pricesData, rankingsData, masterListData] = await Promise.all([
-                        queryPortfolioData(dateStr, null, null, logger).catch(() => null),
-                        queryHistoryData(dateStr, null, null, logger).catch(() => null),
-                        querySocialData(dateStr, null, null, logger).catch(() => null),
-                        queryInstrumentInsights(dateStr, logger).catch(() => null),
-                        queryAssetPrices(dateStr, dateStr, null, logger).catch(() => null), // Check for specific date
-                        queryPIRankings(dateStr, logger).catch(() => null),
-                        queryPIMasterList(logger).catch(() => null)
+                    const [p, h, s, i, pr, r, ml, rt, pv, wl, ah] = await Promise.all([
+                        queryPortfolioData(dateStr, null, null, logger).catch(() => ({})),
+                        queryHistoryData(dateStr, null, null, logger).catch(() => ({})),
+                        querySocialData(dateStr, null, null, logger).catch(() => ({})),
+                        queryInstrumentInsights(dateStr, logger).catch(() => []),
+                        queryAssetPrices(dateStr, dateStr, null, logger).catch(() => ({})),
+                        queryPIRankings(dateStr, logger).catch(() => []),
+                        queryPIMasterList(logger).catch(() => ({})),
+                        queryPIRatings(dateStr, logger).catch(() => ({})),
+                        queryPIPageViews(dateStr, logger).catch(() => ({})),
+                        queryWatchlistMembership(dateStr, logger).catch(() => ({})),
+                        queryPIAlertHistory(dateStr, logger).catch(() => ({}))
                     ]);
-                    
-                    bigqueryHasPortfolio = portfolioData && Object.keys(portfolioData).length > 0;
-                    bigqueryHasHistory = historyData && Object.keys(historyData).length > 0;
-                    bigqueryHasSocial = socialData && Object.keys(socialData).length > 0;
-                    bigqueryHasInsights = insightsData && Array.isArray(insightsData) && insightsData.length > 0;
-                    bigqueryHasPrices = pricesData && Object.keys(pricesData).length > 0;
-                    bigqueryHasPIRankings = rankingsData && Array.isArray(rankingsData) && rankingsData.length > 0;
-                    bigqueryHasPIMasterList = masterListData && Object.keys(masterListData).length > 0;
-                    
-                    if (bigqueryHasPortfolio || bigqueryHasHistory || bigqueryHasSocial || bigqueryHasInsights || bigqueryHasPrices || bigqueryHasPIRankings) {
-                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✅ Found data in BigQuery: portfolio=${bigqueryHasPortfolio}, history=${bigqueryHasHistory}, social=${bigqueryHasSocial}, insights=${bigqueryHasInsights}, prices=${bigqueryHasPrices}, rankings=${bigqueryHasPIRankings}`);
-                    }
-                } catch (bqError) {
-                    logger.log('WARN', `[RootDataIndexer/${dateStr}] BigQuery check failed, using Firestore fallback: ${bqError.message}`);
+                    bqData = { 
+                        portfolio: p || {}, history: h || {}, social: s || {}, 
+                        insights: i || [], prices: pr || {}, rankings: r || [], masterList: ml || {},
+                        ratings: rt || {}, pageViews: pv || {}, watchlists: wl || {}, alerts: ah || {}
+                    };
+                } catch (e) {
+                    logger.log('WARN', `[RootDataIndexer/${dateStr}] BigQuery check failed: ${e.message}`);
                 }
             }
+
+            // Helper to check user types in BQ result maps
+            const hasType = (map, type) => Object.values(map).some(u => u.user_type === type);
+
+            // --- Populate Migrated Flags from BQ ---
+            availability.details.piPortfolios = hasType(bqData.portfolio, 'POPULAR_INVESTOR');
+            availability.details.signedInUserPortfolio = hasType(bqData.portfolio, 'SIGNED_IN_USER');
             
-            // 1. Standard Retail
-            // Path: {normalPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
-            const normPortPartsRef = db.collection(safeCollections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
+            availability.details.piHistory = hasType(bqData.history, 'POPULAR_INVESTOR');
+            availability.details.signedInUserHistory = hasType(bqData.history, 'SIGNED_IN_USER');
             
-            // Path: {speculatorPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
-            const specPortPartsRef = db.collection(safeCollections.speculatorPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
+            availability.details.piSocial = hasType(bqData.social, 'POPULAR_INVESTOR');
+            availability.details.signedInSocial = hasType(bqData.social, 'SIGNED_IN_USER');
+
+            availability.details.piRankings = bqData.rankings.length > 0 || (bqData.rankings.Items && bqData.rankings.Items.length > 0);
+            availability.details.piMasterList = Object.keys(bqData.masterList).length > 0;
+            
+            // Profile Metrics
+            availability.details.piRatings = Object.keys(bqData.ratings).length > 0;
+            availability.details.piPageViews = Object.keys(bqData.pageViews).length > 0;
+            availability.details.watchlistMembership = Object.keys(bqData.watchlists).length > 0;
+            availability.details.piAlertHistory = Object.keys(bqData.alerts).length > 0;
+
+            // Global Flags based on BQ
+            availability.hasInsights = bqData.insights.length > 0;
+            availability.hasPrices = Object.keys(bqData.prices).length > 0;
+
+            // =========================================================================
+            // 2. FIRESTORE CHECKS (Only for Non-Migrated Data)
+            // =========================================================================
             
-            // Path: {normalHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
+            // A. Normal/Speculator Portfolios & History (Legacy Structure)
+            // Path: {collection}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
+            const normPortPartsRef = db.collection(safeCollections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
+            const specPortPartsRef = db.collection(safeCollections.speculatorPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
             const normHistPartsRef = db.collection(safeCollections.normalHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
-            
-            // Path: {speculatorHistory}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
             const specHistPartsRef = db.collection(safeCollections.speculatorHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
-            
-            // Path: {insights}/{YYYY-MM-DD}
-            const insightsRef = db.collection(safeCollections.insights).doc(dateStr);
-            
-            // Generic Asset Posts
+
+            // B. Generic Social (Instrument Feed)
             // Path: {social}/{YYYY-MM-DD}/posts (Limit 1)
             const socialPostsRef = db.collection(safeCollections.social).doc(dateStr).collection('posts');
 
-            // 2. Popular Investors (NEW STRUCTURE)
-            // Path: {piRankings}/{YYYY-MM-DD}
-            const piRankingsRef = db.collection(safeCollections.piRankings).doc(dateStr);
-
-            // Path: PopularInvestorPortfolioData/{YYYY-MM-DD}/{cid}
-            const piPortfoliosCollectionRef = db.collection('PopularInvestorPortfolioData').doc(dateStr);
-            
-            // Path: PopularInvestorTradeHistoryData/{YYYY-MM-DD}/{cid}
-            const piHistoryCollectionRef = db.collection('PopularInvestorTradeHistoryData').doc(dateStr);
-            
-            // Path: PopularInvestorSocialPostData/{YYYY-MM-DD}/{cid}
-            const piSocialCollectionRef = db.collection('PopularInvestorSocialPostData').doc(dateStr);
-            
-            // Legacy paths (for backward compatibility during migration)
-            const piPortfoliosPartsRef = db.collection(safeCollections.piPortfolios)
-                .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
-            
-            const piDeepPartsRef = db.collection(safeCollections.piDeepPortfolios)
-                .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
-            
-            const piHistoryPartsRef = db.collection(safeCollections.piHistory)
-                .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
-            
-            // 3. Signed-In Users (NEW STRUCTURE)
-            // Path: SignedInUserPortfolioData/{YYYY-MM-DD}/{cid}
-            const signedInPortCollectionRef = db.collection('SignedInUserPortfolioData').doc(dateStr);
-            
-            // Path: SignedInUserTradeHistoryData/{YYYY-MM-DD}/{cid}
-            const signedInHistCollectionRef = db.collection('SignedInUserTradeHistoryData').doc(dateStr);
-            
-            // Path: SignedInUserSocialPostData/{YYYY-MM-DD}/{cid}
-            const signedInSocialCollectionRef = db.collection('SignedInUserSocialPostData').doc(dateStr);
-            
-            // Path: {verifications} (Limit 1) - Checks if collection is non-empty generally
+            // C. Verifications
             const verificationsRef = db.collection(safeCollections.verifications);
-            
-            // 4. New Root Data Types for Profile Metrics (Global documents per date)
-            // Path: PIRatingsData/{YYYY-MM-DD}
-            const piRatingsRef = db.collection(safeCollections.piRatings).doc(dateStr);
-            
-            // Path: PIPageViewsData/{YYYY-MM-DD}
-            const piPageViewsRef = db.collection(safeCollections.piPageViews).doc(dateStr);
-            
-            // Path: WatchlistMembershipData/{YYYY-MM-DD}
-            const watchlistMembershipRef = db.collection(safeCollections.watchlistMembership).doc(dateStr);
-            
-            // Path: PIAlertHistoryData/{YYYY-MM-DD}
-            const piAlertHistoryRef = db.collection(safeCollections.piAlertHistory).doc(dateStr);
 
-            // [NEW] Master List Ref (Single Global Document)
-            // Path: system_state/popular_investor_master_list
-            const piMasterListRef = db.collection(safeCollections.piMasterList).doc('popular_investor_master_list');
-
-            // 4. Social Data Checks - Use date tracking documents (NEW STRUCTURE)
-            // Single tracking documents at root level:
-            // - PopularInvestorSocialPostData/_dates -> fetchedDates.{date}
-            // - SignedInUserSocialPostData/_dates -> fetchedDates.{date}
-            // For generic social (InstrumentFeedSocialPostData), check the posts subcollection
-            const [piSocialTrackingDoc, signedInSocialTrackingDoc, genericSocialSnap] = await Promise.all([
-                db.collection('PopularInvestorSocialPostData').doc('_dates').get(),
-                db.collection('SignedInUserSocialPostData').doc('_dates').get(),
-                db.collection('InstrumentFeedSocialPostData').doc(dateStr).collection('posts').limit(1).get()
-            ]);
-            
-            // Check if date exists in tracking documents
-            // The _dates document uses dot notation: fetchedDates.2025-12-29: true
-            // When read, this becomes: { fetchedDates: { "2025-12-29": true } }
-            let foundPISocial = false;
-            let foundSignedInSocial = false;
-            
-            if (piSocialTrackingDoc.exists) {
-                const data = piSocialTrackingDoc.data();
-                // Check both nested structure and flat dot-notation structure
-                if (data.fetchedDates && typeof data.fetchedDates === 'object') {
-                    if (data.fetchedDates[dateStr] === true) {
-                        foundPISocial = true;
-                    }
-                } else if (data[`fetchedDates.${dateStr}`] === true) {
-                    // Handle flat dot-notation structure (if Firestore stores it that way)
-                    foundPISocial = true;
-                }
-            }
-            
-            if (signedInSocialTrackingDoc.exists) {
-                const data = signedInSocialTrackingDoc.data();
-                // Check both nested structure and flat dot-notation structure
-                if (data.fetchedDates && typeof data.fetchedDates === 'object') {
-                    if (data.fetchedDates[dateStr] === true) {
-                        foundSignedInSocial = true;
-                    }
-                } else if (data[`fetchedDates.${dateStr}`] === true) {
-                    // Handle flat dot-notation structure (if Firestore stores it that way)
-                    foundSignedInSocial = true;
-                }
-            } 
-
-            // --- Execute Checks ---
-            // Helper to check if any documents exist in a date collection (new structure)
-            const checkDateCollectionHasDocs = async (collectionRef) => {
-                try {
-                    // For new structure: collection/{date} is a document, check if it has subcollections (CIDs)
-                    const subcollections = await collectionRef.listCollections();
-                    if (subcollections.length === 0) return false;
-                    // Check if any subcollection (CID) has a document
-                    for (const subcol of subcollections) {
-                        const docs = await subcol.limit(1).get();
-                        if (!docs.empty) return true;
-                    }
-                    return false;
-                } catch (e) {
-                    return false;
-                }
-            };
-            
+            // Execute Firestore Queries
             const [
                 normPortExists, specPortExists,
                 normHistExists, specHistExists,
-                insightsSnap, genericSocialExists,
-                piRankingsSnap, 
-                piPortExists,  
-                piDeepExists, 
-                piHistExists, 
-                signedInPortExists, signedInHistExists, 
-                verificationsQuery,
-                // New Root Data Types
-                piRatingsSnap,
-                piPageViewsSnap,
-                watchlistMembershipSnap,
-                piAlertHistorySnap,
-                piMasterListSnap // [NEW]
+                genericSocialDocs,
+                verificationsQuery
             ] = await Promise.all([
                 checkAnyPartExists(normPortPartsRef),
                 checkAnyPartExists(specPortPartsRef),
                 checkAnyPartExists(normHistPartsRef),
                 checkAnyPartExists(specHistPartsRef),
-                insightsRef.get(),
-                Promise.resolve(!genericSocialSnap.empty),
-                piRankingsRef.get(),
-                // Check new structure first, fallback to legacy
-                checkDateCollectionHasDocs(piPortfoliosCollectionRef).then(exists => exists || checkAnyPartExists(piPortfoliosPartsRef)),
-                checkAnyPartExists(piDeepPartsRef), // Legacy only
-                // Check new structure first, fallback to legacy
-                checkDateCollectionHasDocs(piHistoryCollectionRef).then(exists => exists || checkAnyPartExists(piHistoryPartsRef)),
-                // Check new structure for signed-in users
-                checkDateCollectionHasDocs(signedInPortCollectionRef),
-                checkDateCollectionHasDocs(signedInHistCollectionRef),
-                verificationsRef.limit(1).get(),
-                // New Root Data Types - check if document exists
-                piRatingsRef.get(),
-                piPageViewsRef.get(),
-                watchlistMembershipRef.get(),
-                piAlertHistoryRef.get(),
-                piMasterListRef.get() // [NEW] - Moved to end to match destructuring order
-            ]);
-            
-            // Also check social collections directly (new structure)
-            const [piSocialExists, signedInSocialExists] = await Promise.all([
-                checkDateCollectionHasDocs(piSocialCollectionRef),
-                checkDateCollectionHasDocs(signedInSocialCollectionRef)
+                socialPostsRef.limit(1).get(),
+                verificationsRef.limit(1).get()
             ]);
-            
-            // Update social flags if found in new structure
-            if (piSocialExists) foundPISocial = true;
-            if (signedInSocialExists) foundSignedInSocial = true;
-
-            // Social data checks are done above using tracking documents
-            // foundPISocial and foundSignedInSocial are already set from the tracking document checks
-            logger.log('INFO', `[RootDataIndexer/${dateStr}] Social check results - PI: ${foundPISocial}, Signed-in: ${foundSignedInSocial}`);
 
-            // --- Assign to Availability ---
-            // Use BigQuery results if available, otherwise fall back to Firestore
-            availability.details.normalPortfolio     = bigqueryHasPortfolio || normPortExists;
-            availability.details.speculatorPortfolio = bigqueryHasPortfolio || specPortExists;
-            availability.details.normalHistory       = bigqueryHasHistory || normHistExists;
-            availability.details.speculatorHistory   = bigqueryHasHistory || specHistExists;
-            availability.details.piRankings          = bigqueryHasPIRankings || piRankingsSnap.exists;
-            
-            availability.details.piPortfolios        = bigqueryHasPortfolio || piPortExists; 
-            availability.details.piDeepPortfolios    = piDeepExists; // Legacy only, no BigQuery equivalent
-            availability.details.piHistory           = bigqueryHasHistory || piHistExists;
-            
-            // PI & Signed-In Social Flags (Strict)
-            // Use BigQuery if available, otherwise use Firestore tracking documents
-            const finalPISocial = bigqueryHasSocial || foundPISocial;
-            const finalSignedInSocial = bigqueryHasSocial || foundSignedInSocial;
-            availability.details.piSocial            = finalPISocial;
-            availability.details.hasPISocial         = finalPISocial; 
-            availability.details.signedInSocial      = finalSignedInSocial;
-            availability.details.hasSignedInSocial   = finalSignedInSocial; 
-            
-            // Signed-In Flags
-            availability.details.signedInUserPortfolio = bigqueryHasPortfolio || signedInPortExists;
-            availability.details.signedInUserHistory   = bigqueryHasHistory || signedInHistExists;
+            // --- Populate Non-Migrated Flags ---
+            availability.details.normalPortfolio     = normPortExists;
+            availability.details.speculatorPortfolio = specPortExists;
+            availability.details.normalHistory       = normHistExists;
+            availability.details.speculatorHistory   = specHistExists;
             availability.details.signedInUserVerification = !verificationsQuery.empty;
-            
-            // New Root Data Types for Profile Metrics
-            availability.details.piRatings = piRatingsSnap.exists;
-            availability.details.piPageViews = piPageViewsSnap.exists;
-            availability.details.watchlistMembership = watchlistMembershipSnap.exists;
-            availability.details.piAlertHistory = piAlertHistorySnap.exists;
-            
-            // Aggregates (use BigQuery if available, otherwise Firestore)
-            availability.hasPortfolio = bigqueryHasPortfolio || normPortExists || specPortExists || piPortExists || signedInPortExists;
-            availability.hasHistory   = bigqueryHasHistory || normHistExists || specHistExists || piHistExists || signedInHistExists;
-            availability.hasInsights  = bigqueryHasInsights || insightsSnap.exists;
-            availability.hasSocial    = bigqueryHasSocial || finalPISocial || finalSignedInSocial || genericSocialExists;
 
-            // [NEW] Assign Master List Availability (BigQuery first, then Firestore)
-            availability.details.piMasterList = bigqueryHasPIMasterList || piMasterListSnap.exists;
-            
-            // Price Check (BigQuery first, then Firestore tracking document)
-            const hasPriceForDate = bigqueryHasPrices || priceAvailabilitySet.has(dateStr);
-            availability.hasPrices = hasPriceForDate;
+            // =========================================================================
+            // 3. AGGREGATE FLAGS
+            // =========================================================================
             
-            if (targetDate && !hasPriceForDate) {
-                logger.log('WARN', `[RootDataIndexer/${dateStr}] Price data not found in tracking document. Set size: ${priceAvailabilitySet.size}, Date checked: ${dateStr}`);
-            }
+            availability.hasPortfolio = 
+                availability.details.piPortfolios || 
+                availability.details.signedInUserPortfolio || 
+                normPortExists || 
+                specPortExists;
+
+            availability.hasHistory = 
+                availability.details.piHistory || 
+                availability.details.signedInUserHistory || 
+                normHistExists || 
+                specHistExists;
+
+            availability.hasSocial = 
+                availability.details.piSocial || 
+                availability.details.signedInSocial || 
+                !genericSocialDocs.empty;
 
+            // Store Result
             await db.collection(availabilityCollection).doc(dateStr).set(availability);
             updatesCount++;
             
-            // Log detailed results for this date
-            const detailsLog = Object.entries(availability.details)
-                .map(([key, value]) => `${key}: ${value ? '✓' : '✗'}`)
-                .join(', ');
-            logger.log('INFO', `[RootDataIndexer/${dateStr}] Data availability: ${detailsLog}`);
+            const summary = [
+                availability.hasPortfolio ? 'Port' : '',
+                availability.hasHistory ? 'Hist' : '',
+                availability.hasSocial ? 'Soc' : '',
+                availability.hasInsights ? 'Ins' : '',
+                availability.hasPrices ? 'Prc' : ''
+            ].filter(Boolean).join('+');
+
+            logger.log('INFO', `[RootDataIndexer/${dateStr}] Indexed. Found: [${summary || 'NONE'}]`);
 
        } catch (e) {
-            logger.log('ERROR', `[RootDataIndexer] Failed to index ${dateStr}`, {
-                message: e.message,
-                code: e.code
-            });
+            logger.log('ERROR', `[RootDataIndexer] Failed to index ${dateStr}`, { message: e.message });
         }
     }));
 
     await Promise.all(promises);
     
-    // Log appropriately based on results
-    if (updatesCount === 0) {
-        logger.log('WARN', `[RootDataIndexer] Indexing complete but NO dates were updated. This may indicate a failure. Mode: ${scanMode}`, { targetDate });
-    } else {
-        logger.log('SUCCESS', `[RootDataIndexer] Indexing complete. Updated ${updatesCount} dates. Mode: ${scanMode}`, { targetDate, updatesCount });
-    }
+    if (updatesCount === 0) logger.log('WARN', `[RootDataIndexer] No dates updated. Mode: ${scanMode}`);
+    else logger.log('SUCCESS', `[RootDataIndexer] Updated ${updatesCount} dates. Mode: ${scanMode}`);
     
-    return { success: updatesCount > 0, count: updatesCount, mode: scanMode };
+    return { success: updatesCount > 0, count: updatesCount };
 };