bulltrackers-module 1.0.469 → 1.0.471

package/functions/root-data-indexer/index.js

@@ -235,12 +235,38 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             // Path: {verifications} (Limit 1) - Checks if collection is non-empty generally
             const verificationsRef = db.collection(collections.verifications || 'user_verifications');
 
-            // 4. Universal Social Check via Collection Group
-            // Path: Collection Group 'posts' where 'fetchedAt' is within the day
-            const universalSocialQuery = db.collectionGroup('posts')
-                .where('fetchedAt', '>=', dayStart)
-                .where('fetchedAt', '<=', dayEnd)
-                .limit(50); 
+            // 4. Social Data Checks - Use date tracking documents
+            // Single tracking documents at root level:
+            // - pi_social_posts/_dates -> fetchedDates.{date}
+            // - signed_in_users_social/_dates -> fetchedDates.{date}
+            // For generic social (daily_social_insights), check the collection directly
+            const [piSocialTrackingDoc, signedInSocialTrackingDoc, genericSocialSnap] = await Promise.all([
+                db.collection(PI_SOCIAL_COLL_NAME).doc('_dates').get(),
+                db.collection(SIGNED_IN_SOCIAL_COLL_NAME).doc('_dates').get(),
+                db.collectionGroup('posts')
+                    .where('fetchedAt', '>=', dayStart)
+                    .where('fetchedAt', '<=', dayEnd)
+                    .limit(1)
+                    .get()
+            ]);
+            
+            // Check if date exists in tracking documents
+            let foundPISocial = false;
+            let foundSignedInSocial = false;
+            
+            if (piSocialTrackingDoc.exists) {
+                const data = piSocialTrackingDoc.data();
+                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                    foundPISocial = true;
+                }
+            }
+            
+            if (signedInSocialTrackingDoc.exists) {
+                const data = signedInSocialTrackingDoc.data();
+                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                    foundSignedInSocial = true;
+                }
+            } 
 
             // --- Execute Checks ---
             const [
@@ -252,8 +278,7 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 piDeepExists, 
                 piHistExists, 
                 signedInPortExists, signedInHistExists, 
-                verificationsQuery,
-                universalSocialSnap
+                verificationsQuery
             ] = await Promise.all([
                 checkAnyPartExists(normPortPartsRef),
                 checkAnyPartExists(specPortPartsRef),
@@ -267,43 +292,12 @@ exports.runRootDataIndexer = async (config, dependencies) => {
                 checkAnyPartExists(piHistoryPartsRef),
                 checkAnyPartExists(signedInPortPartsRef),
                 checkAnyPartExists(signedInHistPartsRef),
-                verificationsRef.limit(1).get(),
-                universalSocialQuery.get()
+                verificationsRef.limit(1).get()
             ]);
 
-            // --- Evaluate Social Sources ---
-            let foundPISocial = false;
-            let foundSignedInSocial = false;
-
-            if (!universalSocialSnap.empty) {
-                logger.log('DEBUG', `[RootDataIndexer/${dateStr}] Found ${universalSocialSnap.docs.length} social posts in query`);
-                universalSocialSnap.docs.forEach(doc => {
-                    const path = doc.ref.path;
-                    const data = doc.data();
-                    const fetchedAt = data.fetchedAt;
-                    
-                    // Use includes() to match collection name anywhere in path (more robust)
-                    // Path format: {collectionName}/{userId}/posts/{postId}
-                    // Firestore paths don't have leading slash, so check both with and without
-                    const piMatchPattern = `${PI_SOCIAL_COLL_NAME}/`;
-                    const signedInMatchPattern = `${SIGNED_IN_SOCIAL_COLL_NAME}/`;
-                    
-                    if (path.includes(piMatchPattern) || path.startsWith(piMatchPattern)) {
-                        foundPISocial = true;
-                        logger.log('DEBUG', `[RootDataIndexer/${dateStr}] ✓ Found PI social: ${path}`);
-                    }
-                    if (path.includes(signedInMatchPattern) || path.startsWith(signedInMatchPattern)) {
-                        foundSignedInSocial = true;
-                        const fetchedAtStr = fetchedAt ? (fetchedAt.toDate ? fetchedAt.toDate().toISOString() : String(fetchedAt)) : 'missing';
-                        logger.log('DEBUG', `[RootDataIndexer/${dateStr}] ✓ Found signed-in social: ${path}, fetchedAt: ${fetchedAtStr}`);
-                    } else if (!path.includes(piMatchPattern)) {
-                        // Log paths that don't match either pattern to help debug
-                        logger.log('DEBUG', `[RootDataIndexer/${dateStr}] ✗ Path doesn't match: ${path} (looking for: "${signedInMatchPattern}" or "${piMatchPattern}")`);
-                    }
-                });
-            } else {
-                logger.log('DEBUG', `[RootDataIndexer/${dateStr}] No social posts found in query (dayStart: ${dayStart.toISOString()}, dayEnd: ${dayEnd.toISOString()})`);
-            }
+            // Social data checks are done above using tracking documents
+            // foundPISocial and foundSignedInSocial are already set from the tracking document checks
+            logger.log('INFO', `[RootDataIndexer/${dateStr}] Social check results - PI: ${foundPISocial}, Signed-in: ${foundSignedInSocial}`);
 
             // --- Assign to Availability ---
             availability.details.normalPortfolio     = normPortExists;

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -351,6 +351,30 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
             offset += take;
         }
 
+        // Write date tracking document after successful fetch
+        // Single document at root level: /signed_in_users_social/_dates and /pi_social_posts/_dates
+        // Contains a map of all dates that have been fetched
+        if (totalSaved > 0 && (type === 'SIGNED_IN_USER' || type === 'POPULAR_INVESTOR')) {
+            const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
+            let trackingCollectionPath;
+            
+            if (type === 'SIGNED_IN_USER') {
+                trackingCollectionPath = config.signedInUserSocialCollection || 'signed_in_users_social';
+            } else {
+                trackingCollectionPath = config.piSocialCollectionName || 'pi_social_posts';
+            }
+            
+            // Write to a single tracking document at the root of the collection
+            const trackingDocRef = db.collection(trackingCollectionPath).doc('_dates');
+            const trackingData = {
+                [`fetchedDates.${today}`]: true,
+                lastUpdated: FieldValue.serverTimestamp()
+            };
+            
+            await trackingDocRef.set(trackingData, { merge: true });
+            logger.log('INFO', `[SocialTask/${taskId}] Updated date tracking for ${type}: ${today}`);
+        }
+
         logger.log('SUCCESS', `[SocialTask/${taskId}] Process complete for ${cid}. Total stored: ${totalSaved}/${MAX_POSTS_TO_STORE}`);
 
     } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.469",
+  "version": "1.0.471",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [