bulltrackers-module 1.0.364 → 1.0.366

package/functions/computation-system/data/AvailabilityChecker.js

@@ -1,6 +1,8 @@
 /**
  * @fileoverview Checks availability of root data via the Root Data Index.
  * REFACTORED: Fully supports granular flags for PI, Signed-In Users, Rankings, and Verification.
+ * FIXED: Added safety fallback for unknown userTypes to prevent impossible runs.
+ * UPDATED: Added strict social data checking for Popular Investors and Signed-In Users.
  */
 const { normalizeName } = require('../utils/utils');
 
@@ -19,8 +21,14 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
             else if (userType === 'popular_investor') {
                 if (!rootDataStatus.piPortfolios) missing.push('piPortfolios');
                 if (!rootDataStatus.piDeepPortfolios) missing.push('piDeepPortfolios');
-            } else if (userType === 'signed_in_user' && !rootDataStatus.signedInUserPortfolio) missing.push('signedInUserPortfolio');
+            } 
+            else if (userType === 'signed_in_user' && !rootDataStatus.signedInUserPortfolio) missing.push('signedInUserPortfolio');
             else if (userType === 'all' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+            else {
+                // [FIX] Safety Block: If userType doesn't match known types (e.g. typo in metadata), 
+                // enforce the global 'hasPortfolio' check instead of silently allowing it.
+                if (!rootDataStatus.hasPortfolio) missing.push('portfolio (fallback)');
+            }
         }
         else if (dep === 'history') {
             if (userType === 'speculator' && !rootDataStatus.speculatorHistory) missing.push('speculatorHistory');
@@ -28,11 +36,31 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
             else if (userType === 'popular_investor' && !rootDataStatus.piHistory) missing.push('piHistory');
             else if (userType === 'signed_in_user' && !rootDataStatus.signedInUserHistory) missing.push('signedInUserHistory');
             else if (userType === 'all' && !rootDataStatus.hasHistory) missing.push('history');
+            else {
+                 // [FIX] Safety Block for History
+                 if (!rootDataStatus.hasHistory) missing.push('history (fallback)');
+            }
         }
         else if (dep === 'rankings' && !rootDataStatus.piRankings) missing.push('piRankings');
         else if (dep === 'verification' && !rootDataStatus.signedInUserVerification) missing.push('signedInUserVerification');
         else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
-        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        
+        // [UPDATED] Strict Social Data Checking
+        else if (dep === 'social') {
+            if (userType === 'popular_investor') {
+                // Strict check: Must have PI-specific social data
+                if (!rootDataStatus.hasPISocial) missing.push('piSocial');
+            }
+            else if (userType === 'signed_in_user') {
+                // Strict check: Must have Signed-In User specific social data
+                if (!rootDataStatus.hasSignedInSocial) missing.push('signedInSocial');
+            }
+            else {
+                // Default behavior: Check for generic asset/market social data
+                if (!rootDataStatus.hasSocial) missing.push('social');
+            }
+        }
+
         else if (dep === 'price' && !rootDataStatus.hasPrices) missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
@@ -84,7 +112,7 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
                     // Global flags
                     hasPortfolio: !!data.hasPortfolio,
                     hasHistory:   !!data.hasHistory,
-                    hasSocial:    !!data.hasSocial,
+                    hasSocial:    !!data.hasSocial, // Represents generic/asset posts
                     hasInsights:  !!data.hasInsights,
                     hasPrices:    !!data.hasPrices,
 
@@ -101,7 +129,11 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
                     piHistory:                !!details.piHistory,
                     signedInUserPortfolio:    !!details.signedInUserPortfolio,
                     signedInUserHistory:      !!details.signedInUserHistory,
-                    signedInUserVerification: !!details.signedInUserVerification
+                    signedInUserVerification: !!details.signedInUserVerification,
+                    
+                    // [UPDATED] Granular Social Flags
+                    hasPISocial:       !!details.hasPISocial || !!data.hasPISocial,
+                    hasSignedInSocial: !!details.hasSignedInSocial || !!data.hasSignedInSocial
                 },
                 portfolioRefs: null, 
                 historyRefs: null,
@@ -116,7 +148,8 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
                     hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false,
                     speculatorPortfolio: false, normalPortfolio: false, speculatorHistory: false, normalHistory: false,
                     piRankings: false, piPortfolios: false, piDeepPortfolios: false, piHistory: false,
-                    signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false
+                    signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false,
+                    hasPISocial: false, hasSignedInSocial: false
                 }
             };
         }

package/functions/computation-system/executors/StandardExecutor.js

@@ -213,14 +213,14 @@ class StandardExecutor {
         const mappings = await loader.loadMappings();
         const SCHEMAS = mathLayer.SCHEMAS;
 
-        // 1. Load Root Data Dependencies if requested by the manifest
+        // 1. Load Root Data
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        
-        // [FIX] Load full maps for verifications and rankings to pass as global data
         const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
         const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
         
-        const socialData = metadata.rootDataDependencies?.includes('social') ? { today: await loader.loadSocial(dateStr) } : null;
+        // [UPDATED] Load the Partitioned Social Data Container
+        // socialContainer = { generic: {}, pi: { userId: {} }, signedIn: { userId: {} } }
+        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
 
         let chunkSuccess = 0;
         let chunkFailures = 0;
@@ -230,13 +230,9 @@ class StandardExecutor {
             const todayHistory = historyData ? historyData[userId] : null;
             
             // 2. Identify User Type
-            // [FIX] Improved Fallback logic for missing _userType tags (common in raw uploads)
             let actualUserType = todayPortfolio._userType;
-
             if (!actualUserType) {
                 if (todayPortfolio.PublicPositions) {
-                    // Could be SPECULATOR or POPULAR_INVESTOR. Check rankings existence to differentiate.
-                    // Note: 'rankings' is the array of PI objects.
                     const isRanked = rankings && rankings.some(r => String(r.CustomerId) === String(userId));
                     actualUserType = isRanked ? 'POPULAR_INVESTOR' : SCHEMAS.USER_TYPES.SPECULATOR;
                 } else {
@@ -252,20 +248,37 @@ class StandardExecutor {
                  }
             }
             
-            // 4. Resolve specific user data for current iteration
+            // 4. Resolve Contextual Data
             const userVerification = verifications ? verifications[userId] : null;
             const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
 
-            // [FIX] Pass allRankings and allVerifications to ContextFactory
+            // [CRITICAL FIX] Select Specific Social Data to Prevent Skew
+            let effectiveSocialData = null;
+            if (socialContainer) {
+                if (actualUserType === 'POPULAR_INVESTOR') {
+                    // PI Route: Only gets PI-specific posts for this user
+                    // (PIs do not need generic noise usually, or you can merge it if required)
+                    effectiveSocialData = socialContainer.pi[userId] || {}; 
+                } else if (actualUserType === 'SIGNED_IN_USER') {
+                    // Signed-In Route: Only gets Signed-In specific posts for this user
+                    effectiveSocialData = socialContainer.signedIn[userId] || {};
+                } else {
+                    // Normal/Speculator/All Route: Gets Generic Market Posts
+                    effectiveSocialData = socialContainer.generic || {};
+                }
+            }
+
             const context = ContextFactory.buildPerUserContext({ 
                 todayPortfolio, yesterdayPortfolio, todayHistory, userId, 
                 userType: actualUserType, dateStr, metadata, mappings, insights, 
-                socialData, 
+                
+                // Inject the filtered subset
+                socialData: effectiveSocialData ? { today: effectiveSocialData } : null,
+                
                 computedDependencies: computedDeps, previousComputedDependencies: prevDeps, 
                 config, deps,
                 verification: userVerification, 
                 rankings: userRanking,
-                // New Global Injections
                 allRankings: rankings,
                 allVerifications: verifications
             });

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -35,34 +35,22 @@ async function filterActiveTasks(db, date, pass, tasks, logger, forceRun = false
             const data = snap.data();
             const isActive = ['PENDING', 'IN_PROGRESS'].includes(data.status);
 
-            // 1. ZOMBIE CHECK (Recover Stale Locks)
             if (isActive) {
                 const lastActivityTime = data.telemetry?.lastHeartbeat 
                     ? new Date(data.telemetry.lastHeartbeat).getTime() 
                     : (data.startedAt ? new Date(data.startedAt).getTime() : 0);
                 
-                const timeSinceActive = Date.now() - lastActivityTime;
-
-                if (timeSinceActive > STALE_LOCK_THRESHOLD_MS) {
+                if ((Date.now() - lastActivityTime) > STALE_LOCK_THRESHOLD_MS) {
                     if (logger) logger.log('WARN', `[Dispatcher] 🧟 Breaking stale lock for ${taskName}.`);
                     return t; 
                 }
                 return null; 
             }
-
-            // 2. COMPLETED CHECK (Ignore)
-            if (data.status === 'COMPLETED') return null; 
-
-            // 3. FAILED CHECK (Pass through to Route Splitter)
-            // We do NOT filter FAILED tasks here. We pass them to splitRoutes() 
-            // which decides if they get promoted to High-Mem or dropped forever.
-            if (data.status === 'FAILED') {
-                 return t; 
-            }
+            // Note: We do NOT filter COMPLETED here anymore for Sweep. 
+            // If the Orchestrator says it needs to run, we run it.
         }
         return t;
     });
-    
     const results = await Promise.all(checkPromises);
     return results.filter(t => t !== null);
 }
@@ -276,7 +264,6 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
         return { dispatched: 0 };
     }
 
-    // [CRITICAL UPDATE] FILTER FOR SWEEP:
     const validTasks = [];
     for (const task of pending) {
         const name = normalizeName(task.name);
@@ -288,26 +275,22 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
             
             // 1. ACTIVE CHECK: Don't double-dispatch if already running
             if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
-                 // Optional: Check for zombies here if needed, but usually Standard Dispatch handles that.
                  logger.log('INFO', `[Sweep] ⏳ Skipping ${name} - Already IN_PROGRESS.`);
                  continue;
             }
 
-            // 2. COMPLETION CHECK: Don't re-dispatch if already done (Ghost State protection)
-            if (data.status === 'COMPLETED') {
-                 // Only skip if the code hash matches (True Duplicate)
-                 // If hashes differ, it's a legitimate Re-Run that needs to happen.
-                 if (data.hash === task.hash) {
-                      logger.log('WARN', `[Sweep] 🛑 Skipping ${name} - Ledger says COMPLETED (Ghost State).`);
-                      continue;
-                 }
-            }
+            // 2. COMPLETION CHECK (GHOST STATE FIX)
+            // We REMOVED the check that skips if (status === 'COMPLETED' && hash === task.hash).
+            // If we are here, 'analyzeDateExecution' (The Brain) decided this task is NOT done
+            // (likely due to a missing or outdated entry in computation_status).
+            // Even if the Ledger (The Log) says it finished, the system state is inconsistent.
+            // We MUST re-run to repair the Status Index.
             
             const stage = data.error?.stage;
             
             // 3. DETERMINISTIC FAILURE CHECK
             if (['QUALITY_CIRCUIT_BREAKER', 'SEMANTIC_GATE', 'SHARDING_LIMIT_EXCEEDED'].includes(stage)) {
-                 // [FIX] If the code hash has changed, ignore the previous failure and allow retry
+                 // If hash matches, it's the exact same code that failed before. Don't retry in loop.
                  if (data.hash === task.hash) {
                      logger.log('WARN', `[Sweep] 🛑 Skipping deterministic failure for ${name} (${stage}).`);
                      continue;
@@ -315,10 +298,13 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
                  logger.log('INFO', `[Sweep] 🔄 Code Updated for ${name}. Retrying sweep despite previous ${stage}.`);
              }
              
-             // 4. DEAD END CHECK
+             // 4. DEAD END CHECK (High Mem)
              if (data.resourceTier === 'high-mem' && data.status === 'FAILED') {
-                 logger.log('WARN', `[Sweep] 🛑 Skipping ${name} - Already failed on High-Mem.`);
-                 continue;
+                 // If code hasn't changed, don't hammer it.
+                 if (data.hash === task.hash) {
+                     logger.log('WARN', `[Sweep] 🛑 Skipping ${name} - Already failed on High-Mem.`);
+                     continue;
+                 }
              }
         }
         validTasks.push(task);
@@ -345,11 +331,7 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
             dispatchId: currentDispatchId,
             triggerReason: 'SWEEP_RECOVERY',
             resources: 'high-mem', // FORCE
-            traceContext: {
-                traceId: traceId,
-                spanId: spanId,
-                sampled: true
-            }
+            traceContext: { traceId, spanId, sampled: true }
         };
     });
 

package/functions/computation-system/utils/data_loader.js

@@ -185,85 +185,77 @@ async function loadDailyInsights(config, deps, dateString) {
     }
 }
 
-/** Stage 5: Load daily social post insights (Comprehensive) */
+/** Stage 5: Load and Partition Social Data */
 async function loadDailySocialPostInsights(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
-    // 1. Standard Instrument posts
-    const collectionName = config.socialInsightsCollectionName || 'daily_social_insights';
+    logger.log('INFO', `Loading and partitioning social data for ${dateString}`);
     
-    logger.log('INFO', `Loading social post insights for ${dateString}`);
-    
-    const postsMap = {};
-
-    const fetchCollection = async (colName) => {
-        try {
-            const postsCollectionRef = db.collection(colName).doc(dateString).collection('posts');
-            const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${colName}/${dateString})`);
-            if (!querySnapshot.empty) {
-                querySnapshot.forEach(doc => { 
-                    postsMap[doc.id] = tryDecompress(doc.data()); 
-                });
-            }
-        } catch (error) {
-            logger.log('WARN', `Failed to load social posts from ${colName}: ${error.message}`);
-        }
+    // 1. Initialize Buckets
+    const result = {
+        generic: {},            // Map<PostId, Data> - For Normal/Speculator
+        pi: {},                 // Map<UserId, Map<PostId, Data>> - For Popular Investors
+        signedIn: {}            // Map<UserId, Map<PostId, Data>> - For Signed-In Users
     };
 
-    // Load Instrument/General posts
-    await fetchCollection(collectionName);
+    const PI_COL_NAME = config.piSocialCollectionName || 'pi_social_posts';
+    const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users';
 
-    // [FIX] Load Per-User Posts (Popular Investors & Signed-In Users)
-    // These are stored as: Collection/{userId}/posts/{postId}
-    // We must find posts created within the target date.
-    
-    const fetchUserPosts = async (parentCollectionName) => {
-         try {
-             // Listing all users is heavy, but required if we don't have a Group Index.
-             // Assumption: Number of PIs/Signed-in users is manageable (e.g. < 5000)
-             // Optimization: If a 'lastPostAt' field exists on user doc, use it to skip.
-             
-             const userDocs = await db.collection(parentCollectionName).listDocuments();
-             logger.log('INFO', `Scanning ${userDocs.length} users in ${parentCollectionName} for posts on ${dateString}`);
-
-             const startDate = new Date(dateString + 'T00:00:00Z');
-             const endDate = new Date(dateString + 'T23:59:59Z');
-
-             // Parallelize in chunks
-             const batchSize = 20;
-             for (let i = 0; i < userDocs.length; i += batchSize) {
-                 const batch = userDocs.slice(i, i + batchSize);
-                 await Promise.all(batch.map(async (userDoc) => {
-                     const postsQuery = userDoc.collection('posts')
-                        .where('createdAt', '>=', startDate.toISOString())
-                        .where('createdAt', '<=', endDate.toISOString());
-                     
-                     const postsSnap = await postsQuery.get();
-                     if (!postsSnap.empty) {
-                         postsSnap.forEach(doc => {
-                             postsMap[doc.id] = tryDecompress(doc.data());
-                         });
-                     }
-                 }));
-             }
+    // 2. Define Time Range (UTC Day)
+    const startDate = new Date(dateString + 'T00:00:00Z');
+    const endDate = new Date(dateString + 'T23:59:59Z');
 
-         } catch (error) {
-             logger.log('WARN', `Failed to scan user posts in ${parentCollectionName}: ${error.message}`);
-         }
-    };
+    try {
+        // 3. Fetch ALL with CollectionGroup
+        // NOTE: Requires Firestore Index: CollectionId 'posts', Field 'fetchedAt' (ASC/DESC)
+        const postsQuery = db.collectionGroup('posts')
+            .where('fetchedAt', '>=', startDate)
+            .where('fetchedAt', '<=', endDate);
 
-    // Load from PI specific social collection
-    const piSocialCol = config.piSocialCollectionName || 'popular_investor_social_posts'; // Updated name per review
-    const signedInSocialCol = config.signedInUserSocialCollection || 'signed_in_users';
+        const querySnapshot = await withRetry(() => postsQuery.get(), `getSocialPosts(${dateString})`);
+        
+        if (!querySnapshot.empty) {
+            querySnapshot.forEach(doc => { 
+                const data = tryDecompress(doc.data());
+                const path = doc.ref.path; 
+                
+                // 4. Partition Logic based on Path
+                if (path.includes(PI_COL_NAME)) {
+                    // Path format: .../pi_social_posts/{userId}/posts/{postId}
+                    const parts = path.split('/');
+                    const colIndex = parts.indexOf(PI_COL_NAME);
+                    if (colIndex !== -1 && parts[colIndex + 1]) {
+                        const userId = parts[colIndex + 1];
+                        if (!result.pi[userId]) result.pi[userId] = {};
+                        result.pi[userId][doc.id] = data;
+                    }
+                } 
+                else if (path.includes(SIGNED_IN_COL_NAME)) {
+                    // Path format: .../signed_in_users/{userId}/posts/{postId}
+                    const parts = path.split('/');
+                    const colIndex = parts.indexOf(SIGNED_IN_COL_NAME);
+                    if (colIndex !== -1 && parts[colIndex + 1]) {
+                        const userId = parts[colIndex + 1];
+                        if (!result.signedIn[userId]) result.signedIn[userId] = {};
+                        result.signedIn[userId][doc.id] = data;
+                    }
+                } 
+                else {
+                    // Default: Generic Instrument Posts
+                    result.generic[doc.id] = data;
+                }
+            });
+            logger.log('INFO', `Loaded Social Data: ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} SignedIn.`);
+        } else {
+            logger.log('WARN', `No social posts found for ${dateString} via CollectionGroup.`);
+        }
 
-    await Promise.all([
-        fetchUserPosts(piSocialCol),
-        fetchUserPosts(signedInSocialCol)
-    ]);
+    } catch (error) {
+        logger.log('ERROR', `Failed to load social posts: ${error.message}`);
+    }
 
-    logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} total social post insights`);
-    return postsMap;
+    return result;
 }
 
 /** Stage 6: Get history part references for a given date */

package/functions/computation-system/utils/utils.js

@@ -1,6 +1,6 @@
 /**
  * FILENAME: computation-system/utils/utils.js
- * UPDATED: Removed hardcoded date logic. Earliest dates are now fully data-driven.
+ * UPDATED: Added undefined sanitization to prevent Firestore write errors.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
@@ -52,6 +52,35 @@ async function withRetry(fn, operationName, maxRetries = 3) {
     }
 }
 
+/** * Recursively sanitizes data for Firestore compatibility.
+ * Converts 'undefined' to 'null' to prevent validation errors.
+ */
+function sanitizeForFirestore(data) {
+    if (data === undefined) return null;
+    if (data === null) return null;
+    if (data instanceof Date) return data;
+    // Handle Firestore Types if necessary (e.g. GeoPoint, Timestamp), passed through as objects usually.
+    
+    if (Array.isArray(data)) {
+        return data.map(item => sanitizeForFirestore(item));
+    }
+    
+    if (typeof data === 'object') {
+        // Check for specific Firestore types that shouldn't be traversed like plain objects
+        if (data.constructor && (data.constructor.name === 'DocumentReference' || data.constructor.name === 'Timestamp')) {
+            return data;
+        }
+
+        const sanitized = {};
+        for (const [key, value] of Object.entries(data)) {
+            sanitized[key] = sanitizeForFirestore(value);
+        }
+        return sanitized;
+    }
+    
+    return data;
+}
+
 /** Commit a batch of writes in chunks. */
 async function commitBatchInChunks(config, deps, writes, operationName) {
     const { db, logger } = deps;
@@ -74,10 +103,17 @@ async function commitBatchInChunks(config, deps, writes, operationName) {
             if (currentOpsCount + 1 > MAX_BATCH_OPS) await commitAndReset();
             currentBatch.delete(write.ref); currentOpsCount++; continue;
         }
+        
+        // [FIX] Sanitize data to remove 'undefined' values before calculating size or setting
+        const safeData = sanitizeForFirestore(write.data);
+        
         let docSize = 100;
-        try { if (write.data) docSize = JSON.stringify(write.data).length; } catch (e) { }
+        try { if (safeData) docSize = JSON.stringify(safeData).length; } catch (e) { }
+        
         if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) await commitAndReset();
-        currentBatch.set(write.ref, write.data, write.options || { merge: true });
+        
+        // Use safeData instead of write.data
+        currentBatch.set(write.ref, safeData, write.options || { merge: true });
         currentOpsCount++; currentBytesEst += docSize;
     }
     await commitAndReset();
@@ -126,5 +162,5 @@ async function getEarliestDataDates(config, deps) {
 module.exports = { 
     FieldValue, FieldPath, normalizeName, commitBatchInChunks, 
     getExpectedDateStrings, getEarliestDataDates, generateCodeHash,
-    generateDataHash, withRetry, DEFINITIVE_EARLIEST_DATES 
+    generateDataHash, withRetry, DEFINITIVE_EARLIEST_DATES, sanitizeForFirestore
 };

package/functions/root-data-indexer/index.js

@@ -1,7 +1,8 @@
 /**
  * @fileoverview Root Data Indexer
  * Runs daily to index exactly what data is available for every date.
- * UPDATED: Includes explicit checks for Signed-In User Portfolios AND History via Snapshots.
+ * UPDATED: Includes explicit checks for Signed-In User Portfolios, History, AND Social Data.
+ * FIXED: 'hasSocial' is now strictly for Generic Asset Data. PI/User data is separated.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -20,6 +21,11 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     } = config;
 
     const PRICE_COLLECTION_NAME = 'asset_prices';
+    
+    // Collection Names (Fail-safe defaults)
+    const PI_SOCIAL_COLL_NAME      = collections.piSocial || 'pi_social_posts';
+    const SIGNED_IN_SOCIAL_COLL_NAME = collections.signedInUserSocialCollection || 'signed_in_users';
+
     logger.log('INFO', '[RootDataIndexer] Starting Root Data Availability Scan...');
     
     // 1. Price Availability
@@ -38,7 +44,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             });
         }
     } catch (e) {
-        logger.log('ERROR', `[RootDataIndexer] Failed to load price shard.`, { error: e.message });
+        logger.log('ERROR', '[RootDataIndexer] Failed to load price shard.', { error: e.message });
     }
 
     // 2. Determine Date Range
@@ -55,6 +61,11 @@ exports.runRootDataIndexer = async (config, dependencies) => {
 
     const promises = datesToScan.map(dateStr => limit(async () => {
         try {
+            // Define Time Range for Social Query (Full Day UTC)
+            const dayStart = new Date(dateStr);
+            const dayEnd = new Date(dateStr);
+            dayEnd.setHours(23, 59, 59, 999);
+
             const availability = {
                 date: dateStr,
                 lastUpdated: FieldValue.serverTimestamp(),
@@ -72,21 +83,24 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                     piPortfolios:        false, 
                     piDeepPortfolios:    false,
                     piHistory:           false, 
-                    piSocial:            false,
+                    piSocial:            false, // Specific Flag for PI
                     signedInUserPortfolio: false,
-                    signedInUserHistory: false, // [NEW]
-                    signedInUserVerification: false 
+                    signedInUserHistory:   false, 
+                    signedInUserVerification: false,
+                    signedInSocial:        false  // Specific Flag for Signed-In
                 }
             };
 
             // --- Define Refs ---
             
-            // 1. Standard Retail
+            // 1. Standard Retail (Date-based)
             const normPortRef = db.collection(collections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
             const specPortRef = db.collection(collections.speculatorPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
             const normHistRef = db.collection(collections.normalHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
             const specHistRef = db.collection(collections.speculatorHistory).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
             const insightsRef = db.collection(collections.insights).doc(dateStr);
+            
+            // Generic Asset Posts
             const socialPostsRef = db.collection(collections.social).doc(dateStr).collection('posts');
 
             // 2. Popular Investors
@@ -95,8 +109,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             const piDeepRef = db.collection(collections.piDeepPortfolios || 'pi_portfolios_deep').doc(dateStr).collection('portfolios');
             const piHistoryRef = db.collection(collections.piHistory || 'pi_trade_history').doc(dateStr).collection('history');
             
-            // 3. Signed-In Users (New Snapshots Structure)
-            // We verify existence by checking the '19M' block parts which BatchManager writes to.
+            // 3. Signed-In Users
             const signedInPortRef = db.collection(collections.signedInUsers || 'signed_in_user_portfolios')
                 .doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts').doc(CANARY_PART_ID);
             
@@ -105,13 +118,20 @@ exports.runRootDataIndexer = async (config, dependencies) => {
 
             const verificationsRef = db.collection(collections.verifications || 'user_verifications');
 
+            // 4. Universal Social Check via Collection Group
+            const universalSocialQuery = db.collectionGroup('posts')
+                .where('fetchedAt', '>=', dayStart)
+                .where('fetchedAt', '<=', dayEnd)
+                .limit(50); 
+
             // --- Execute Checks ---
             const [
                 normPortSnap, specPortSnap,
                 normHistSnap, specHistSnap,
                 insightsSnap, socialQuerySnap,
                 piRankingsSnap, piPortQuery, piDeepQuery, piHistQuery,
-                signedInPortSnap, signedInHistSnap, verificationsQuery
+                signedInPortSnap, signedInHistSnap, verificationsQuery,
+                universalSocialSnap
             ] = await Promise.all([
                 normPortRef.get(), specPortRef.get(),
                 normHistRef.get(), specHistRef.get(),
@@ -122,10 +142,23 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 piHistoryRef.limit(1).get(),
                 signedInPortRef.get(),
                 signedInHistRef.get(),
-                verificationsRef.limit(1).get()
+                verificationsRef.limit(1).get(),
+                universalSocialQuery.get()
             ]);
 
-            // --- Evaluate & Assign ---
+            // --- Evaluate Social Sources ---
+            let foundPISocial = false;
+            let foundSignedInSocial = false;
+
+            if (!universalSocialSnap.empty) {
+                universalSocialSnap.docs.forEach(doc => {
+                    const path = doc.ref.path;
+                    if (path.startsWith(PI_SOCIAL_COLL_NAME)) foundPISocial = true;
+                    if (path.startsWith(SIGNED_IN_SOCIAL_COLL_NAME)) foundSignedInSocial = true;
+                });
+            }
+
+            // --- Assign to Availability ---
             availability.details.normalPortfolio     = normPortSnap.exists;
             availability.details.speculatorPortfolio = specPortSnap.exists;
             availability.details.normalHistory       = normHistSnap.exists;
@@ -135,7 +168,13 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.details.piDeepPortfolios    = !piDeepQuery.empty;
             availability.details.piHistory           = !piHistQuery.empty;
             
-            // [NEW] Signed-In Flags
+            // PI & Signed-In Social Flags (Strict)
+            availability.details.piSocial            = foundPISocial;
+            availability.details.hasPISocial         = foundPISocial; 
+            availability.details.signedInSocial      = foundSignedInSocial;
+            availability.details.hasSignedInSocial   = foundSignedInSocial; 
+
+            // Signed-In Flags
             availability.details.signedInUserPortfolio = signedInPortSnap.exists;
             availability.details.signedInUserHistory   = signedInHistSnap.exists;
             availability.details.signedInUserVerification = !verificationsQuery.empty;
@@ -144,7 +183,12 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.hasPortfolio = normPortSnap.exists || specPortSnap.exists || !piPortQuery.empty || signedInPortSnap.exists;
             availability.hasHistory   = normHistSnap.exists || specHistSnap.exists || !piHistQuery.empty || signedInHistSnap.exists;
             availability.hasInsights  = insightsSnap.exists;
+            
+            // [CRITICAL FIX]
+            // hasSocial strictly refers to GENERIC/ASSET social data.
+            // It does NOT include PI or SignedIn data (they have their own flags).
             availability.hasSocial    = !socialQuerySnap.empty; 
+            
             availability.hasPrices    = priceAvailabilitySet.has(dateStr);
 
             await db.collection(availabilityCollection).doc(dateStr).set(availability);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.364",
+  "version": "1.0.366",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [