bulltrackers-module 1.0.610 → 1.0.612

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -2212,6 +2212,50 @@ const checkDataStatus = async (db, userId) => {
     };
 };
 
+/**
+ * Check Popular Investor data status - finds the latest computation date for a PI
+ * Similar to checkDataStatus but specifically for PopularInvestorProfileMetrics
+ */
+const checkPopularInvestorDataStatus = async (db, piId) => {
+    const lookbackDays = 7;
+    const today = new Date();
+    let computationDate = null;
+    
+    const computationName = 'PopularInvestorProfileMetrics';
+    
+    // Check for computation results in the last 7 days
+    for (let i = 0; i < lookbackDays; i++) {
+        const checkDate = new Date(today);
+        checkDate.setDate(checkDate.getDate() - i);
+        const dateStr = checkDate.toISOString().split('T')[0];
+        
+        try {
+            const pageRef = db.collection('unified_insights')
+                .doc(dateStr)
+                .collection('results')
+                .doc('popular-investor')
+                .collection('computations')
+                .doc(computationName)
+                .collection('pages')
+                .doc(String(piId));
+            
+            const pageSnap = await pageRef.get();
+            if (pageSnap.exists) {
+                computationDate = dateStr;
+                break;
+            }
+        } catch (error) {
+            // Continue checking other dates
+            console.error(`Error checking computation ${computationName} for ${dateStr}:`, error);
+        }
+    }
+    
+    return {
+        computationDate: computationDate,
+        available: computationDate !== null
+    };
+};
+
 const sendTestAlert = async (db, userId, payload) => {
     // Create a fake notification
     const id = `test_${Date.now()}`;
@@ -2539,6 +2583,7 @@ module.exports = {
     getSyncStatus,
     autoGenerateWatchlist,
     checkDataStatus,
+    checkPopularInvestorDataStatus,
     sendTestAlert,
     isSignedInUser,
     getUserUsername,

package/functions/api-v2/routes/popular_investors.js

@@ -147,11 +147,16 @@ router.get('/:piId/profile', async (req, res) => {
         const computationName = 'PopularInvestorProfileMetrics';
         const profileData = await pageCollection(db, targetDate, computationName, piId, parseInt(lookback));
         
+        // Extract computationDate from the latest data entry (first item in array, sorted by date descending)
+        // The profileData array contains { date, data } objects, sorted with latest first
+        const computationDate = profileData && profileData.length > 0 ? profileData[0].date : null;
+        
         res.json({ 
             success: true, 
             computation: computationName,
             piId: piId,
             data: profileData,
+            computationDate: computationDate, // Latest computation date for up-to-date validation
             profileType: 'public' // Indicates this is a public PI profile
         });
     } catch (error) {

package/functions/api-v2/routes/sync.js

@@ -6,7 +6,9 @@ const {
     isDeveloper,
     isSignedInUser,
     fetchPopularInvestorMasterList,
-    getUserUsername
+    getUserUsername,
+    checkDataStatus,
+    checkPopularInvestorDataStatus
 } = require('../helpers/data-fetchers/firestore.js');
 
 const router = express.Router();
@@ -25,20 +27,56 @@ const handleSyncRequest = async (req, res) => {
         const limit = await checkSyncRateLimits(db, targetId, req.targetUserId, isDev);
         if (!limit.allowed) return res.status(429).json({ error: limit.message });
 
-        // 2. Detect User Types
+        // 2. Detect User Types (needed for validation)
         const [isSignedIn, isPI] = await Promise.all([
             isSignedInUser(db, targetId),
             fetchPopularInvestorMasterList(db, String(targetId)).then(() => true).catch(() => false)
         ]);
 
+        // 3. Check if data is already up-to-date (prevent unnecessary syncs)
+        // Developers can bypass this check
+        if (!isDev) {
+            try {
+                let computationDate = null;
+                
+                // Check Popular Investor data status if target is a PI (anyone can sync a PI, so always check)
+                if (isPI) {
+                    const piDataStatus = await checkPopularInvestorDataStatus(db, String(targetId));
+                    computationDate = piDataStatus.computationDate;
+                }
+                
+                // Check signed-in user data status if target is a signed-in user syncing themselves
+                // (Only check if not already found from PI check, and only if syncing themselves)
+                if (isSignedIn && targetId === req.targetUserId && !computationDate) {
+                    const userDataStatus = await checkDataStatus(db, String(targetId));
+                    computationDate = userDataStatus.computationDate;
+                }
+                
+                // If we found a computation date and it's today, block the sync
+                if (computationDate) {
+                    const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD format
+                    if (computationDate === today) {
+                        return res.status(400).json({ 
+                            error: "Data is already up-to-date for today. Sync is not needed.",
+                            code: "DATA_UP_TO_DATE",
+                            computationDate: computationDate
+                        });
+                    }
+                }
+            } catch (error) {
+                // If data status check fails, log but don't block sync (fail open for reliability)
+                console.warn(`[handleSyncRequest] Failed to check data status for ${targetId}:`, error.message);
+            }
+        }
+
         if (!isSignedIn && !isPI) {
             return res.status(404).json({ error: "User not found in SignedInUsers or Popular Investor master list" });
         }
 
-        // 3. Get username
+        // 4. Get username
         const username = await getUserUsername(db, targetId) || String(targetId);
 
-        // 4. Create request IDs and dispatch tasks
+        // 5. Create request IDs and dispatch tasks
         const requestIds = [];
         const tasks = [];
 

package/functions/computation-system/persistence/ResultCommitter.js

@@ -15,6 +15,7 @@ const ContractValidator                    = require('./ContractValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
 const zlib                                 = require('zlib'); 
+const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils');
 
 const NON_RETRYABLE_ERRORS = [ 'PERMISSION_DENIED', 'DATA_LOSS', 'FAILED_PRECONDITION' ];
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
@@ -137,6 +138,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
+            // [NEW] Page Computation Logic (Fan-Out) with TTL
+            // Bypasses standard compression/sharding to write per-user documents
             // [NEW] Page Computation Logic (Fan-Out) with TTL
             // Bypasses standard compression/sharding to write per-user documents
             if (isPageComputation && !isEmpty) {
@@ -145,26 +148,27 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     .collection(config.computationsSubcollection).doc(name);
 
                 // --- CLEANUP START: Remove old storage formats (Sharded/Compressed) ---
-                // If the computation previously ran in Standard mode, we must remove shards
-                // and clear the main document data to avoid conflicting flags.
-                try {
-                    const docSnap = await mainDocRef.get();
-                    if (docSnap.exists) {
-                        const dData = docSnap.data();
-                        if (dData._sharded) {
-                            const shardCol = mainDocRef.collection('_shards');
-                            const shardDocs = await withRetry(() => shardCol.listDocuments());
-                            
-                            if (shardDocs.length > 0) {
-                                const cleanupOps = shardDocs.map(d => ({ type: 'DELETE', ref: d }));
-                                await commitBatchInChunks(config, deps, cleanupOps, `${name}::PageModeCleanup`);
-                                runMetrics.io.deletes += cleanupOps.length;
-                                logger.log('INFO', `[PageMode] ${name}: Cleaned up ${cleanupOps.length} old shard documents.`);
+                // Optimization: Only attempt cleanup on the initial write to save reads
+                if (isInitialWrite) {
+                    try {
+                        const docSnap = await mainDocRef.get();
+                        if (docSnap.exists) {
+                            const dData = docSnap.data();
+                            if (dData._sharded) {
+                                const shardCol = mainDocRef.collection('_shards');
+                                const shardDocs = await withRetry(() => shardCol.listDocuments());
+                                
+                                if (shardDocs.length > 0) {
+                                    const cleanupOps = shardDocs.map(d => ({ type: 'DELETE', ref: d }));
+                                    await commitBatchInChunks(config, deps, cleanupOps, `${name}::PageModeCleanup`);
+                                    runMetrics.io.deletes += cleanupOps.length;
+                                    logger.log('INFO', `[PageMode] ${name}: Cleaned up ${cleanupOps.length} old shard documents.`);
+                                }
                             }
                         }
+                    } catch (cleanupErr) {
+                        logger.log('WARN', `[PageMode] ${name}: Cleanup warning: ${cleanupErr.message}`);
                     }
-                } catch (cleanupErr) {
-                    logger.log('WARN', `[PageMode] ${name}: Cleanup warning: ${cleanupErr.message}`);
                 }
                 // --- CLEANUP END ---
 
@@ -198,31 +202,47 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     logger.log('INFO', `[PageMode] ${name}: Wrote ${pageWrites.length} user pages. TTL: ${ttlDays}d.`);
                 }
 
-                // 3. Write the "Header" document (Important for Status/Metrics/TTL)
-                if (flushMode !== 'INTERMEDIATE') {
-                    const headerData = {
-                        _completed: true,
-                        _isPageMode: true, // Flag for readers to know where to look
-                        _pageCount: pageWrites.length,
-                        _lastUpdated: new Date().toISOString(),
-                        _expireAt: expireAt // Ensure the header also gets deleted
-                    };
-                    
-                    // CHANGED: Use merge: false to FORCE overwrite. 
-                    // This clears old "payload" (compressed), raw data keys, and old flags like "_sharded".
-                    await mainDocRef.set(headerData, { merge: false });
-                    runMetrics.io.writes += 1;
-                    
-                    if (calc.manifest.hash) { 
-                        successUpdates[name] = { 
-                            hash: calc.manifest.hash, 
-                            simHash: simHash, 
-                            resultHash: resultHash,
-                            category: calc.manifest.category, 
-                            composition: calc.manifest.composition, 
-                            metrics: runMetrics 
-                        }; 
-                    }
+                // 3. Write or Update the "Header" document
+                // FIXED: Now runs on every batch to ensure counts are accumulated correctly.
+                
+                const isFinalFlush = (flushMode !== 'INTERMEDIATE');
+                
+                // Determine Page Count Value: Raw number for initial, Increment for updates
+                let pageCountValue = pageWrites.length;
+                if (!isInitialWrite) {
+                    pageCountValue = FieldValue.increment(pageWrites.length);
+                }
+
+                const headerData = {
+                    _isPageMode: true, // Flag for readers to know where to look
+                    _pageCount: pageCountValue,
+                    _lastUpdated: new Date().toISOString(),
+                    _expireAt: expireAt // Ensure the header also gets deleted
+                };
+
+                // Handle Completion Status
+                if (isFinalFlush) {
+                    headerData._completed = true;
+                } else if (isInitialWrite) {
+                    headerData._completed = false; // Initialize as incomplete
+                }
+
+                // Write Strategy:
+                // isInitialWrite = TRUE  -> merge: false (Wipes old Standard Mode data/schema)
+                // isInitialWrite = FALSE -> merge: true (Updates count and status, preserves data)
+                await mainDocRef.set(headerData, { merge: !isInitialWrite });
+                
+                runMetrics.io.writes += 1;
+                
+                if (isFinalFlush && calc.manifest.hash) { 
+                    successUpdates[name] = { 
+                        hash: calc.manifest.hash, 
+                        simHash: simHash, 
+                        resultHash: resultHash,
+                        category: calc.manifest.category, 
+                        composition: calc.manifest.composition, 
+                        metrics: runMetrics 
+                    }; 
                 }
                 
                 continue; // Skip the standard writeSingleResult logic

package/functions/computation-system/utils/utils.js

@@ -6,6 +6,7 @@
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const crypto                    = require('crypto');
 
+
 // [UPDATED] Registry for Data Availability. 
 // Populated dynamically by getEarliestDataDates().
 const DEFINITIVE_EARLIEST_DATES = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.610",
+  "version": "1.0.612",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [