bulltrackers-module 1.0.440 → 1.0.441

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -275,17 +275,52 @@ async function getPiFetchStatus(req, res, dependencies, config) {
         
         if (!requestsSnapshot.empty) {
             const latestRequest = requestsSnapshot.docs[0].data();
-            const status = latestRequest.status || 'queued';
+            let status = latestRequest.status || 'queued';
+            
+            // If status is 'indexing' or 'computing', check if computation results are now available
+            if (status === 'indexing' || status === 'computing') {
+                // Re-check if computation results exist
+                const checkDate = new Date();
+                for (let i = 0; i < 2; i++) { // Check today and yesterday
+                    const dateStr = new Date(checkDate);
+                    dateStr.setDate(checkDate.getDate() - i);
+                    const dateStrFormatted = dateStr.toISOString().split('T')[0];
+                    
+                    const docRef = db.collection(insightsCollection)
+                        .doc(dateStrFormatted)
+                        .collection(resultsSub)
+                        .doc('popular-investor')
+                        .collection(compsSub)
+                        .doc('PopularInvestorProfileMetrics');
+                    
+                    const doc = await docRef.get();
+                    if (doc.exists) {
+                        const { tryDecompress } = require('./data_helpers');
+                        const data = tryDecompress(doc.data());
+                        
+                        if (data && data[String(piCidNum)]) {
+                            // Computation completed! Update status
+                            status = 'completed';
+                            await requestsSnapshot.docs[0].ref.update({
+                                status: 'completed',
+                                completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                                updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                            });
+                            break;
+                        }
+                    }
+                }
+            }
             
             const response = {
                 success: true,
-                dataAvailable: false,
+                dataAvailable: status === 'completed',
                 status,
                 requestId: latestRequest.requestId,
                 startedAt: latestRequest.startedAt?.toDate?.()?.toISOString() || null,
                 createdAt: latestRequest.createdAt?.toDate?.()?.toISOString() || null,
                 estimatedCompletion: latestRequest.startedAt 
-                    ? new Date(new Date(latestRequest.startedAt.toDate()).getTime() + 5 * 60 * 1000).toISOString()
+                    ? new Date(new Date(latestRequest.startedAt.toDate()).getTime() + 10 * 60 * 1000).toISOString() // 10 min for computation
                     : null
             };
             
@@ -295,6 +330,12 @@ async function getPiFetchStatus(req, res, dependencies, config) {
                 response.failedAt = latestRequest.failedAt?.toDate?.()?.toISOString() || null;
             }
             
+            // Include raw data status if computing
+            if (status === 'computing') {
+                response.rawDataStoredAt = latestRequest.rawDataStoredAt?.toDate?.()?.toISOString() || null;
+                response.message = 'Raw data stored, computation in progress...';
+            }
+            
             return res.status(200).json(response);
         }
         

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -19,7 +19,7 @@ const { shouldTryProxy, recordProxyOutcome, getFailureCount, getMaxFailures } =
  * @param {object} dependencies - db, logger, proxyManager, batchManager, headerManager.
  */
 async function handlePopularInvestorUpdate(taskData, config, dependencies) {
-    const { logger, proxyManager, batchManager, headerManager, db } = dependencies;
+    const { logger, proxyManager, batchManager, headerManager, db, pubsub } = dependencies;
     
     // Validate taskData exists and has required fields
     if (!taskData) {
@@ -299,7 +299,7 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
 
         logger.log('SUCCESS', `[PI Update] Completed full update for ${username}`);
         
-        // Update request status to completed if this is an on-demand request
+        // Update request status and trigger computation if this is an on-demand request
         if (requestId && source === 'on_demand' && db) {
             try {
                 const requestRef = db.collection('pi_fetch_requests')
@@ -307,13 +307,82 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
                     .collection('requests')
                     .doc(requestId);
                 
+                // Update status to indicate raw data is stored, indexing and computation will be triggered
                 await requestRef.update({
-                    status: 'completed',
-                    completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    status: 'indexing',
+                    rawDataStoredAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
                     updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
                 });
+                
+                // CRITICAL: Trigger root data indexer FIRST so computation system knows data exists
+                try {
+                    const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                    const indexerConfig = {
+                        availabilityCollection: config.rootDataIndexer?.availabilityCollection || 'system_root_data_index',
+                        earliestDate: config.rootDataIndexer?.earliestDate || '2025-08-01',
+                        collections: config.rootDataIndexer?.collections || {},
+                        targetDate: today // Index only today's date for speed
+                    };
+                    
+                    logger.log('INFO', `[PI Update] Triggering root data indexer for date ${today} before computation...`);
+                    await runRootDataIndexer(indexerConfig, dependencies);
+                    logger.log('INFO', `[PI Update] Root data indexer completed for date ${today}`);
+                    
+                    // Update status to indicate indexing is done, computation is being triggered
+                    await requestRef.update({
+                        status: 'computing',
+                        indexedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                    });
+                } catch (indexerError) {
+                    logger.log('ERROR', `[PI Update] Failed to run root data indexer for ${today}`, indexerError);
+                    // Continue anyway - computation might still work if index already exists
+                    // But update status to indicate we tried
+                    await requestRef.update({
+                        status: 'computing',
+                        indexerError: indexerError.message,
+                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                    });
+                }
+                
+                // Trigger computation for PopularInvestorProfileMetrics
+                const { pubsub } = dependencies;
+                if (pubsub) {
+                    const computationTopic = config.computationSystem?.computationTopicStandard || 'computation-tasks';
+                    const topic = pubsub.topic(computationTopic);
+                    const crypto = require('crypto');
+                    
+                    const computationMessage = {
+                        action: 'RUN_COMPUTATION_DATE',
+                        computation: 'PopularInvestorProfileMetrics',
+                        date: today,
+                        pass: '1', // Standard pass
+                        dispatchId: crypto.randomUUID(),
+                        triggerReason: 'on_demand_pi_fetch',
+                        resources: 'standard',
+                        metadata: {
+                            onDemand: true,
+                            requestId: requestId,
+                            piCid: cid,
+                            piUsername: username
+                        },
+                        traceContext: {
+                            traceId: crypto.randomBytes(16).toString('hex'),
+                            spanId: crypto.randomBytes(8).toString('hex'),
+                            sampled: true
+                        }
+                    };
+                    
+                    await topic.publishMessage({
+                        data: Buffer.from(JSON.stringify(computationMessage))
+                    });
+                    
+                    logger.log('INFO', `[PI Update] Triggered computation PopularInvestorProfileMetrics for PI ${cid} (${username}) for date ${today}`);
+                } else {
+                    logger.log('WARN', `[PI Update] PubSub not available, cannot trigger computation`);
+                }
             } catch (err) {
-                logger.log('WARN', `[PI Update] Failed to update request status to completed for ${requestId}`, err);
+                logger.log('WARN', `[PI Update] Failed to update request status or trigger computation for ${requestId}`, err);
             }
         }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.440",
+  "version": "1.0.441",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [