bulltrackers-module 1.0.439 → 1.0.441

package/functions/generic-api/user-api/helpers/data_helpers.js

@@ -2483,6 +2483,7 @@ module.exports = {
     checkIfUserIsPopularInvestor,
     checkIfUserIsPI, // Export for use in review_helpers
     findLatestRankingsDate, // Export for use in on_demand_fetch_helpers
+    tryDecompress, // Export for use in on_demand_fetch_helpers
     trackProfileView,
     getSignedInUserPIPersonalizedMetrics
 };

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -275,17 +275,52 @@ async function getPiFetchStatus(req, res, dependencies, config) {
         
         if (!requestsSnapshot.empty) {
             const latestRequest = requestsSnapshot.docs[0].data();
-            const status = latestRequest.status || 'queued';
+            let status = latestRequest.status || 'queued';
+            
+            // If status is 'indexing' or 'computing', check if computation results are now available
+            if (status === 'indexing' || status === 'computing') {
+                // Re-check if computation results exist
+                const checkDate = new Date();
+                for (let i = 0; i < 2; i++) { // Check today and yesterday
+                    const dateStr = new Date(checkDate);
+                    dateStr.setDate(checkDate.getDate() - i);
+                    const dateStrFormatted = dateStr.toISOString().split('T')[0];
+                    
+                    const docRef = db.collection(insightsCollection)
+                        .doc(dateStrFormatted)
+                        .collection(resultsSub)
+                        .doc('popular-investor')
+                        .collection(compsSub)
+                        .doc('PopularInvestorProfileMetrics');
+                    
+                    const doc = await docRef.get();
+                    if (doc.exists) {
+                        const { tryDecompress } = require('./data_helpers');
+                        const data = tryDecompress(doc.data());
+                        
+                        if (data && data[String(piCidNum)]) {
+                            // Computation completed! Update status
+                            status = 'completed';
+                            await requestsSnapshot.docs[0].ref.update({
+                                status: 'completed',
+                                completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                                updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                            });
+                            break;
+                        }
+                    }
+                }
+            }
             
             const response = {
                 success: true,
-                dataAvailable: false,
+                dataAvailable: status === 'completed',
                 status,
                 requestId: latestRequest.requestId,
                 startedAt: latestRequest.startedAt?.toDate?.()?.toISOString() || null,
                 createdAt: latestRequest.createdAt?.toDate?.()?.toISOString() || null,
                 estimatedCompletion: latestRequest.startedAt 
-                    ? new Date(new Date(latestRequest.startedAt.toDate()).getTime() + 5 * 60 * 1000).toISOString()
+                    ? new Date(new Date(latestRequest.startedAt.toDate()).getTime() + 10 * 60 * 1000).toISOString() // 10 min for computation
                     : null
             };
             
@@ -295,6 +330,12 @@ async function getPiFetchStatus(req, res, dependencies, config) {
                 response.failedAt = latestRequest.failedAt?.toDate?.()?.toISOString() || null;
             }
             
+            // Include raw data status if computing
+            if (status === 'computing') {
+                response.rawDataStoredAt = latestRequest.rawDataStoredAt?.toDate?.()?.toISOString() || null;
+                response.message = 'Raw data stored, computation in progress...';
+            }
+            
             return res.status(200).json(response);
         }
         

package/functions/task-engine/handler_creator.js

@@ -154,12 +154,26 @@ async function handleRequest(message, context, config, dependencies) {
                 await handleUpdate(data, 'single-update', dependencies, config); 
                 break;
             case 'POPULAR_INVESTOR_UPDATE':
-                // Validate data exists before passing to handler
-                if (!data) {
-                    logger.log('ERROR', '[TaskEngine] POPULAR_INVESTOR_UPDATE received with no data', { payload });
+                // For POPULAR_INVESTOR_UPDATE, the entire payload IS the task data
+                // (not wrapped in a 'data' field like other task types)
+                // Extract task data from payload, excluding 'type'
+                const taskData = data || {
+                    cid: payload.cid,
+                    username: payload.username,
+                    requestId: payload.requestId,
+                    source: payload.source,
+                    requestedBy: payload.requestedBy,
+                    actualRequestedBy: payload.actualRequestedBy,
+                    metadata: payload.metadata,
+                    priority: payload.priority
+                };
+                
+                if (!taskData || (!taskData.cid && !taskData.username)) {
+                    logger.log('ERROR', '[TaskEngine] POPULAR_INVESTOR_UPDATE missing required fields (cid or username)', { payload, taskData });
                     return;
                 }
-                await handlePopularInvestorUpdate(data, config, dependencies);
+                
+                await handlePopularInvestorUpdate(taskData, config, dependencies);
                 break;
             case 'ON_DEMAND_USER_UPDATE':
                 const onDemandData = data || payload;

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -19,7 +19,7 @@ const { shouldTryProxy, recordProxyOutcome, getFailureCount, getMaxFailures } =
  * @param {object} dependencies - db, logger, proxyManager, batchManager, headerManager.
  */
 async function handlePopularInvestorUpdate(taskData, config, dependencies) {
-    const { logger, proxyManager, batchManager, headerManager, db } = dependencies;
+    const { logger, proxyManager, batchManager, headerManager, db, pubsub } = dependencies;
     
     // Validate taskData exists and has required fields
     if (!taskData) {
@@ -299,7 +299,7 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
 
         logger.log('SUCCESS', `[PI Update] Completed full update for ${username}`);
         
-        // Update request status to completed if this is an on-demand request
+        // Update request status and trigger computation if this is an on-demand request
         if (requestId && source === 'on_demand' && db) {
             try {
                 const requestRef = db.collection('pi_fetch_requests')
@@ -307,13 +307,82 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
                     .collection('requests')
                     .doc(requestId);
                 
+                // Update status to indicate raw data is stored, indexing and computation will be triggered
                 await requestRef.update({
-                    status: 'completed',
-                    completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    status: 'indexing',
+                    rawDataStoredAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
                     updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
                 });
+                
+                // CRITICAL: Trigger root data indexer FIRST so computation system knows data exists
+                try {
+                    const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                    const indexerConfig = {
+                        availabilityCollection: config.rootDataIndexer?.availabilityCollection || 'system_root_data_index',
+                        earliestDate: config.rootDataIndexer?.earliestDate || '2025-08-01',
+                        collections: config.rootDataIndexer?.collections || {},
+                        targetDate: today // Index only today's date for speed
+                    };
+                    
+                    logger.log('INFO', `[PI Update] Triggering root data indexer for date ${today} before computation...`);
+                    await runRootDataIndexer(indexerConfig, dependencies);
+                    logger.log('INFO', `[PI Update] Root data indexer completed for date ${today}`);
+                    
+                    // Update status to indicate indexing is done, computation is being triggered
+                    await requestRef.update({
+                        status: 'computing',
+                        indexedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                    });
+                } catch (indexerError) {
+                    logger.log('ERROR', `[PI Update] Failed to run root data indexer for ${today}`, indexerError);
+                    // Continue anyway - computation might still work if index already exists
+                    // But update status to indicate we tried
+                    await requestRef.update({
+                        status: 'computing',
+                        indexerError: indexerError.message,
+                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                    });
+                }
+                
+                // Trigger computation for PopularInvestorProfileMetrics
+                const { pubsub } = dependencies;
+                if (pubsub) {
+                    const computationTopic = config.computationSystem?.computationTopicStandard || 'computation-tasks';
+                    const topic = pubsub.topic(computationTopic);
+                    const crypto = require('crypto');
+                    
+                    const computationMessage = {
+                        action: 'RUN_COMPUTATION_DATE',
+                        computation: 'PopularInvestorProfileMetrics',
+                        date: today,
+                        pass: '1', // Standard pass
+                        dispatchId: crypto.randomUUID(),
+                        triggerReason: 'on_demand_pi_fetch',
+                        resources: 'standard',
+                        metadata: {
+                            onDemand: true,
+                            requestId: requestId,
+                            piCid: cid,
+                            piUsername: username
+                        },
+                        traceContext: {
+                            traceId: crypto.randomBytes(16).toString('hex'),
+                            spanId: crypto.randomBytes(8).toString('hex'),
+                            sampled: true
+                        }
+                    };
+                    
+                    await topic.publishMessage({
+                        data: Buffer.from(JSON.stringify(computationMessage))
+                    });
+                    
+                    logger.log('INFO', `[PI Update] Triggered computation PopularInvestorProfileMetrics for PI ${cid} (${username}) for date ${today}`);
+                } else {
+                    logger.log('WARN', `[PI Update] PubSub not available, cannot trigger computation`);
+                }
             } catch (err) {
-                logger.log('WARN', `[PI Update] Failed to update request status to completed for ${requestId}`, err);
+                logger.log('WARN', `[PI Update] Failed to update request status or trigger computation for ${requestId}`, err);
             }
         }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.439",
+  "version": "1.0.441",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [