bulltrackers-module 1.0.470 → 1.0.471

package/functions/root-data-indexer/index.js

@@ -235,16 +235,38 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             // Path: {verifications} (Limit 1) - Checks if collection is non-empty generally
             const verificationsRef = db.collection(collections.verifications || 'user_verifications');
 
-            // 4. Universal Social Check via Collection Group
-            // Path: Collection Group 'posts' where 'fetchedAt' is within the day
-            // This queries ALL 'posts' subcollections across:
-            // - signed_in_users_social/{userId}/posts/{postId}
-            // - pi_social_posts/{userId}/posts/{postId}
-            // - daily_social_insights/{date}/posts/{postId}
-            const universalSocialQuery = db.collectionGroup('posts')
-                .where('fetchedAt', '>=', dayStart)
-                .where('fetchedAt', '<=', dayEnd)
-                .limit(100); // Increased limit to ensure we catch all posts 
+            // 4. Social Data Checks - Use date tracking documents
+            // Single tracking documents at root level:
+            // - pi_social_posts/_dates -> fetchedDates.{date}
+            // - signed_in_users_social/_dates -> fetchedDates.{date}
+            // For generic social (daily_social_insights), check the collection directly
+            const [piSocialTrackingDoc, signedInSocialTrackingDoc, genericSocialSnap] = await Promise.all([
+                db.collection(PI_SOCIAL_COLL_NAME).doc('_dates').get(),
+                db.collection(SIGNED_IN_SOCIAL_COLL_NAME).doc('_dates').get(),
+                db.collectionGroup('posts')
+                    .where('fetchedAt', '>=', dayStart)
+                    .where('fetchedAt', '<=', dayEnd)
+                    .limit(1)
+                    .get()
+            ]);
+            
+            // Check if date exists in tracking documents
+            let foundPISocial = false;
+            let foundSignedInSocial = false;
+            
+            if (piSocialTrackingDoc.exists) {
+                const data = piSocialTrackingDoc.data();
+                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                    foundPISocial = true;
+                }
+            }
+            
+            if (signedInSocialTrackingDoc.exists) {
+                const data = signedInSocialTrackingDoc.data();
+                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                    foundSignedInSocial = true;
+                }
+            } 
 
             // --- Execute Checks ---
             const [
@@ -256,8 +278,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 piDeepExists, 
                 piHistExists, 
                 signedInPortExists, signedInHistExists, 
-                verificationsQuery,
-                universalSocialSnap
+                verificationsQuery
             ] = await Promise.all([
                 checkAnyPartExists(normPortPartsRef),
                 checkAnyPartExists(specPortPartsRef),
@@ -271,73 +292,12 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 checkAnyPartExists(piHistoryPartsRef),
                 checkAnyPartExists(signedInPortPartsRef),
                 checkAnyPartExists(signedInHistPartsRef),
-                verificationsRef.limit(1).get(),
-                universalSocialQuery.get()
+                verificationsRef.limit(1).get()
             ]);
 
-            // --- Evaluate Social Sources ---
-            let foundPISocial = false;
-            let foundSignedInSocial = false;
-
-            logger.log('INFO', `[RootDataIndexer/${dateStr}] Checking social data availability (dayStart: ${dayStart.toISOString()}, dayEnd: ${dayEnd.toISOString()})`);
-            logger.log('INFO', `[RootDataIndexer/${dateStr}] Looking for PI social in: "${PI_SOCIAL_COLL_NAME}", Signed-in social in: "${SIGNED_IN_SOCIAL_COLL_NAME}"`);
-
-            if (!universalSocialSnap.empty) {
-                logger.log('INFO', `[RootDataIndexer/${dateStr}] Collection group query returned ${universalSocialSnap.docs.length} posts`);
-                
-                // Track all paths found for debugging
-                const allPaths = [];
-                const piPaths = [];
-                const signedInPaths = [];
-                const otherPaths = [];
-                
-                universalSocialSnap.docs.forEach(doc => {
-                    const path = doc.ref.path;
-                    const data = doc.data();
-                    const fetchedAt = data.fetchedAt;
-                    allPaths.push(path);
-                    
-                    // Convert fetchedAt to string for logging
-                    let fetchedAtStr = 'missing';
-                    if (fetchedAt) {
-                        if (fetchedAt.toDate) {
-                            fetchedAtStr = fetchedAt.toDate().toISOString();
-                        } else if (fetchedAt.toMillis) {
-                            fetchedAtStr = new Date(fetchedAt.toMillis()).toISOString();
-                        } else {
-                            fetchedAtStr = String(fetchedAt);
-                        }
-                    }
-                    
-                    // Use includes() to match collection name anywhere in path
-                    // Path format: {collectionName}/{userId}/posts/{postId}
-                    const piMatchPattern = `${PI_SOCIAL_COLL_NAME}/`;
-                    const signedInMatchPattern = `${SIGNED_IN_SOCIAL_COLL_NAME}/`;
-                    
-                    if (path.includes(piMatchPattern)) {
-                        foundPISocial = true;
-                        piPaths.push({ path, fetchedAt: fetchedAtStr });
-                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✓ PI social: ${path} (fetchedAt: ${fetchedAtStr})`);
-                    } else if (path.includes(signedInMatchPattern)) {
-                        foundSignedInSocial = true;
-                        signedInPaths.push({ path, fetchedAt: fetchedAtStr });
-                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✓ Signed-in social: ${path} (fetchedAt: ${fetchedAtStr})`);
-                    } else {
-                        otherPaths.push({ path, fetchedAt: fetchedAtStr });
-                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ? Other social: ${path} (fetchedAt: ${fetchedAtStr})`);
-                    }
-                });
-                
-                logger.log('INFO', `[RootDataIndexer/${dateStr}] Summary - Total: ${allPaths.length}, PI: ${piPaths.length}, Signed-in: ${signedInPaths.length}, Other: ${otherPaths.length}`);
-                
-                if (signedInPaths.length === 0 && !foundSignedInSocial) {
-                    logger.log('WARN', `[RootDataIndexer/${dateStr}] ⚠️ No signed-in social posts found! Expected pattern: "${signedInMatchPattern}"`);
-                    logger.log('WARN', `[RootDataIndexer/${dateStr}] All paths found: ${allPaths.slice(0, 10).join(', ')}${allPaths.length > 10 ? '...' : ''}`);
-                }
-            } else {
-                logger.log('WARN', `[RootDataIndexer/${dateStr}] ⚠️ Collection group query returned NO posts! (dayStart: ${dayStart.toISOString()}, dayEnd: ${dayEnd.toISOString()})`);
-                logger.log('WARN', `[RootDataIndexer/${dateStr}] This might indicate: 1) No posts with fetchedAt in this range, 2) Missing Firestore index, or 3) Query error`);
-            }
+            // Social data checks are done above using tracking documents
+            // foundPISocial and foundSignedInSocial are already set from the tracking document checks
+            logger.log('INFO', `[RootDataIndexer/${dateStr}] Social check results - PI: ${foundPISocial}, Signed-in: ${foundSignedInSocial}`);
 
             // --- Assign to Availability ---
             availability.details.normalPortfolio     = normPortExists;

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -351,6 +351,30 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
             offset += take;
         }
 
+        // Write date tracking document after successful fetch
+        // Single document at root level: /signed_in_users_social/_dates and /pi_social_posts/_dates
+        // Contains a map of all dates that have been fetched
+        if (totalSaved > 0 && (type === 'SIGNED_IN_USER' || type === 'POPULAR_INVESTOR')) {
+            const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
+            let trackingCollectionPath;
+            
+            if (type === 'SIGNED_IN_USER') {
+                trackingCollectionPath = config.signedInUserSocialCollection || 'signed_in_users_social';
+            } else {
+                trackingCollectionPath = config.piSocialCollectionName || 'pi_social_posts';
+            }
+            
+            // Write to a single tracking document at the root of the collection
+            const trackingDocRef = db.collection(trackingCollectionPath).doc('_dates');
+            const trackingData = {
+                [`fetchedDates.${today}`]: true,
+                lastUpdated: FieldValue.serverTimestamp()
+            };
+            
+            await trackingDocRef.set(trackingData, { merge: true });
+            logger.log('INFO', `[SocialTask/${taskId}] Updated date tracking for ${type}: ${today}`);
+        }
+
         logger.log('SUCCESS', `[SocialTask/${taskId}] Process complete for ${cid}. Total stored: ${totalSaved}/${MAX_POSTS_TO_STORE}`);
 
     } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.470",
+  "version": "1.0.471",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [