bulltrackers-module 1.0.480 → 1.0.491

package/functions/computation-system/helpers/computation_worker.js

@@ -231,6 +231,28 @@ async function handleComputationTask(message, config, dependencies) {
 
         await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
         await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
+        
+        // Send notification if this was an on-demand computation
+        if (metadata?.onDemand && metadata?.requestId && metadata?.requestingUserCid) {
+            try {
+                const { notifyComputationComplete, getComputationDisplayName } = require('../../generic-api/user-api/helpers/notification_helpers');
+                await notifyComputationComplete(
+                    dependencies.db,
+                    dependencies.logger,
+                    metadata.requestingUserCid,
+                    metadata.requestId,
+                    computation,
+                    getComputationDisplayName(computation),
+                    true,
+                    null
+                );
+            } catch (notifError) {
+                // Non-critical, log and continue
+                if (dependencies.logger) {
+                    dependencies.logger.log('WARN', `[Worker] Failed to send completion notification for ${computation}`, notifError);
+                }
+            }
+        }
 
     } catch (err) { 
         heartbeats.stop();
@@ -257,6 +279,29 @@ async function handleComputationTask(message, config, dependencies) {
             }, { merge: true });
 
             await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeats.getPeak() }, triggerReason, resourceTier);
+            
+            // Send error notification if this was an on-demand computation
+            if (metadata?.onDemand && metadata?.requestId && metadata?.requestingUserCid) {
+                try {
+                    const { notifyComputationComplete, getComputationDisplayName } = require('../../generic-api/user-api/helpers/notification_helpers');
+                    await notifyComputationComplete(
+                        db,
+                        dependencies.logger,
+                        metadata.requestingUserCid,
+                        metadata.requestId,
+                        computation,
+                        getComputationDisplayName(computation),
+                        false,
+                        err.message
+                    );
+                } catch (notifError) {
+                    // Non-critical, log and continue
+                    if (dependencies.logger) {
+                        dependencies.logger.log('WARN', `[Worker] Failed to send error notification for ${computation}`, notifError);
+                    }
+                }
+            }
+            
             return;
         }
         

package/functions/generic-api/user-api/helpers/FALLBACK_CONDITIONS.md

@@ -0,0 +1,98 @@
+# Profile Page Fallback Conditions
+
+## Overview
+
+This document explains when and why the profile page uses fallback mode instead of computation data.
+
+## Data Storage Locations
+
+### Signed-In User Computations
+- **Category**: `popular-investor` (NOT `signed_in_user`)
+- **Reason**: ResultCommitter ignores category metadata when computation is in a non-core folder
+- **Computation**: `SignedInUserProfileMetrics` is in `calculations/popular-investor/` folder
+- **Path**: `/unified_insights/{date}/results/popular-investor/computations/SignedInUserProfileMetrics`
+
+### Signed-In User Raw Data
+- **Portfolio**: `signed_in_users/{blockId}/snapshots/{date}/parts/part_*`
+- **History**: `signed_in_user_history/{blockId}/snapshots/{date}/parts/part_*` (or `signed_in_users_history` depending on config)
+- **Social**: `signed_in_users_social/{cid}/posts/{postId}`
+
+## Fallback Conditions
+
+### Frontend Fallback Logic (UserProfile.tsx)
+
+The profile page uses fallback mode when:
+
+1. **Computation Not Found**: 
+   - API call to `/user/me/computations?computation=SignedInUserProfileMetrics` returns empty `data: {}`
+   - OR the computation exists but doesn't contain data for the user's CID
+
+2. **API Error**:
+   - Any error fetching the computation (network error, 500, etc.)
+
+3. **Data Structure Issues**:
+   - Computation exists but `metricsResponse.data[latestDate]?.SignedInUserProfileMetrics` is undefined/null
+
+### Backend Fallback Logic (getUserComputations)
+
+The API returns `isFallback: false` but `data: {}` when:
+
+1. **Wrong Category Lookup** (FIXED):
+   - Was looking in `signed_in_user` category
+   - Should look in `popular-investor` category
+
+2. **No Data for User**:
+   - Computation document exists but doesn't contain `data[String(effectiveCid)]`
+   - This can happen if:
+     - Computation ran but user wasn't included (shouldn't happen with targetCid optimization)
+     - Data is sharded and shard doesn't contain user's data
+     - Data exists but in wrong format
+
+3. **Date Mismatch**:
+   - Computation exists for different date than requested
+   - `mode=latest` tries to find latest date, but if none found, returns empty
+
+## Current Issue
+
+The user reported:
+- API returns `"isFallback": false, "data": {}`
+- Profile page shows fallback mode
+- Computation exists at `/unified_insights/2025-12-29/results/popular-investor/computations/SignedInUserProfileMetrics/_shards/shard_0`
+
+**Root Cause**: The `getUserComputations` endpoint was looking in the wrong category (`signed_in_user` instead of `popular-investor`).
+
+**Fix Applied**: Updated `getUserComputations` to look in `popular-investor` category for `SignedInUserProfileMetrics`.
+
+## Verification Steps
+
+1. Check computation exists:
+   ```
+   /unified_insights/2025-12-29/results/popular-investor/computations/SignedInUserProfileMetrics
+   ```
+
+2. Check if data is sharded:
+   - If `_sharded: true`, check `_shards` subcollection
+   - Merge all shards to get full data
+
+3. Check if user's CID exists in data:
+   - Data structure: `{ "29312236": { ...metrics... } }`
+   - Must use `String(cid)` as key
+
+4. Check API response:
+   - Should return `data: { "2025-12-29": { "SignedInUserProfileMetrics": { ... } } }`
+   - If empty, check logs for why lookup failed
+
+## History Data Storage
+
+Signed-in user trade history is stored in:
+- Collection: `signed_in_user_history` (or `signed_in_users_history` depending on config)
+- Path: `{collection}/{blockId}/snapshots/{date}/parts/part_*`
+- Block ID: Typically `19M` (canary block)
+
+The task engine stores history via:
+```javascript
+await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
+```
+
+This uses `getHistoryCollection('signed_in_user')` which returns `signedInHistory` collection.
+

package/functions/generic-api/user-api/helpers/HISTORY_STORAGE_LOCATION.md

@@ -0,0 +1,66 @@
+# Signed-In User History Storage Location
+
+## Current Issue
+
+History data is being stored in the **wrong collection** because `FIRESTORE_COLLECTION_SIGNED_IN_HISTORY` is not defined in the task engine config, causing it to fall back to `NormalUserTradeHistory`.
+
+## Storage Path Structure
+
+For signed-in users, history data should be stored at:
+```
+{signed_in_user_history}/{blockId}/snapshots/{date}/parts/part_{shardIndex}
+```
+
+Where:
+- `signed_in_user_history` = Collection name (default: `signed_in_user_history`)
+- `blockId` = `'19M'` (canary block ID)
+- `date` = Date string in format `YYYY-MM-DD` (e.g., `2025-12-29`)
+- `shardIndex` = `cid % TOTAL_SHARDS` (modulo sharding for even distribution)
+
+## Example Path
+
+For user CID `29312236` on date `2025-12-29`:
+```
+signed_in_user_history/19M/snapshots/2025-12-29/parts/part_{shardIndex}
+```
+
+Where `shardIndex = 29312236 % TOTAL_SHARDS` (typically 7 shards, so `29312236 % 7 = 1`)
+
+## Fix Applied
+
+Added `FIRESTORE_COLLECTION_SIGNED_IN_HISTORY` to `taskEngine_config.js`:
+```javascript
+FIRESTORE_COLLECTION_SIGNED_IN_HISTORY: getEnvVar('FIRESTORE_COLLECTION_SIGNED_IN_HISTORY', 'signed_in_user_history'),
+```
+
+## Before Fix
+
+Without this config, the batch manager was using:
+- `config.FIRESTORE_COLLECTION_SIGNED_IN_HISTORY` = `undefined`
+- Fallback: `config.FIRESTORE_COLLECTION_NORMAL_HISTORY` = `'NormalUserTradeHistory'`
+
+So data was being stored at:
+```
+NormalUserTradeHistory/19M/snapshots/2025-12-29/parts/part_{shardIndex}
+```
+
+## After Fix
+
+With the config added, data will be stored at:
+```
+signed_in_user_history/19M/snapshots/2025-12-29/parts/part_{shardIndex}
+```
+
+## Verification
+
+To verify history data exists for a user:
+1. Calculate shard index: `userCid % 7` (or whatever TOTAL_SHARDS is)
+2. Check path: `signed_in_user_history/19M/snapshots/2025-12-29/parts/part_{shardIndex}`
+3. Look for user's CID as a key in the document: `{ "29312236": { ...historyData... } }`
+
+## Related Collections
+
+- **Portfolio**: `signed_in_users/19M/snapshots/{date}/parts/part_{shardIndex}`
+- **History**: `signed_in_user_history/19M/snapshots/{date}/parts/part_{shardIndex}` (FIXED)
+- **Social**: `signed_in_users_social/{cid}/posts/{postId}`
+

package/functions/generic-api/user-api/helpers/data_helpers.js

@@ -870,8 +870,10 @@ async function getUserComputations(req, res, dependencies, config) {
         const resultsSub = config.resultsSubcollection || 'results';
         const compsSub = config.computationsSubcollection || 'computations';
         
-        // For signed-in users, computations are stored in the 'signed_in_user' category
-        const category = 'signed_in_user';
+        // NOTE: ResultCommitter ignores category metadata if computation is in a non-core folder
+        // SignedInUserProfileMetrics is in popular-investor folder, so it's stored in popular-investor category
+        // For signed-in users, computations are stored in the 'popular-investor' category (not 'signed_in_user')
+        const category = 'popular-investor';
         const today = new Date().toISOString().split('T')[0];
         
         const computationNames = computation ? computation.split(',') : [];

package/functions/generic-api/user-api/helpers/notification_helpers.js

@@ -0,0 +1,131 @@
+/**
+ * @fileoverview Notification Helpers for On-Demand Requests
+ * Sends notifications to users when their on-demand sync/computation requests complete
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+
+/**
+ * Send a notification to a user about their on-demand request
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} logger - Logger instance
+ * @param {number} userCid - User CID to notify
+ * @param {string} type - Notification type ('success', 'error', 'progress')
+ * @param {string} title - Notification title
+ * @param {string} message - Notification message
+ * @param {Object} metadata - Additional metadata (requestId, computationName, etc.)
+ */
+async function sendOnDemandNotification(db, logger, userCid, type, title, message, metadata = {}) {
+    try {
+        const notificationId = `ondemand_${Date.now()}_${userCid}_${Math.random().toString(36).substring(2, 9)}`;
+        const notificationRef = db.collection('user_notifications')
+            .doc(String(userCid))
+            .collection('notifications')
+            .doc(notificationId);
+        
+        const notificationData = {
+            id: notificationId,
+            type: 'on_demand',
+            subType: type, // 'success', 'error', 'progress'
+            title,
+            message,
+            read: false,
+            createdAt: FieldValue.serverTimestamp(),
+            metadata: {
+                ...metadata,
+                userCid: Number(userCid)
+            }
+        };
+        
+        await notificationRef.set(notificationData);
+        
+        // Update notification counter
+        const today = new Date().toISOString().split('T')[0];
+        const counterRef = db.collection('user_notifications')
+            .doc(String(userCid))
+            .collection('counters')
+            .doc(today);
+        
+        await counterRef.set({
+            date: today,
+            unreadCount: FieldValue.increment(1),
+            totalCount: FieldValue.increment(1),
+            lastUpdated: FieldValue.serverTimestamp()
+        }, { merge: true });
+        
+        logger.log('INFO', `[sendOnDemandNotification] Sent ${type} notification to user ${userCid}: ${title}`);
+        
+    } catch (error) {
+        logger.log('ERROR', `[sendOnDemandNotification] Failed to send notification to user ${userCid}`, error);
+        // Don't throw - notifications are non-critical
+    }
+}
+
+/**
+ * Send notification when task engine completes data fetch
+ */
+async function notifyTaskEngineComplete(db, logger, requestingUserCid, requestId, username, success, error = null) {
+    if (!requestingUserCid) return; // No user to notify
+    
+    const type = success ? 'success' : 'error';
+    const title = success 
+        ? 'Data Sync Complete' 
+        : 'Data Sync Failed';
+    const message = success
+        ? `Your data sync for ${username} has completed. Portfolio, history, and social data have been stored.`
+        : (error || 'An error occurred while syncing your data. Please try again.');
+    
+    await sendOnDemandNotification(db, logger, requestingUserCid, type, title, message, {
+        requestId,
+        username,
+        stage: 'task_engine',
+        success
+    });
+}
+
+/**
+ * Send notification when computation completes
+ */
+async function notifyComputationComplete(db, logger, requestingUserCid, requestId, computationName, displayName, success, error = null) {
+    if (!requestingUserCid) return; // No user to notify
+    
+    const type = success ? 'success' : 'error';
+    const title = success
+        ? 'Computation Complete'
+        : 'Computation Failed';
+    const message = success
+        ? `${displayName || computationName} has been computed and stored.`
+        : (error || `Failed to compute ${displayName || computationName}. Please try again.`);
+    
+    await sendOnDemandNotification(db, logger, requestingUserCid, type, title, message, {
+        requestId,
+        computationName,
+        displayName,
+        stage: 'computation',
+        success
+    });
+}
+
+/**
+ * Get human-readable name for a computation
+ */
+function getComputationDisplayName(computationName) {
+    const displayNames = {
+        'SignedInUserProfileMetrics': 'Profile Metrics',
+        'SignedInUserCopiedList': 'Copied Investors List',
+        'SignedInUserCopiedPIs': 'Copied Popular Investors',
+        'SignedInUserPastCopies': 'Past Copy History',
+        'PopularInvestorProfileMetrics': 'Popular Investor Profile',
+        'SignedInUserPIPersonalizedMetrics': 'Personalized Metrics'
+    };
+    
+    return displayNames[computationName] || computationName;
+}
+
+module.exports = {
+    sendOnDemandNotification,
+    notifyTaskEngineComplete,
+    notifyComputationComplete,
+    getComputationDisplayName
+};
+

package/functions/generic-api/user-api/helpers/user_sync_helpers.js

@@ -163,11 +163,16 @@ async function requestUserSync(req, res, dependencies, config) {
             requestId,
             requestedBy: Number(requestingUserCid),
             effectiveRequestedBy: effectiveCid,
+            data: {
+                includeSocial: true, // Always include social data for on-demand syncs
+                since: new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString() // Last 7 days
+            },
             metadata: {
                 onDemand: true,
                 targetCid: targetCidNum, // Target specific user for optimization
                 requestedAt: now.toISOString(),
-                isImpersonating: isImpersonating || false
+                isImpersonating: isImpersonating || false,
+                requestingUserCid: Number(requestingUserCid) // Store for notifications
             }
         };
         
@@ -245,12 +250,15 @@ async function getUserSyncStatus(req, res, dependencies, config) {
                 const compsSub = config.computationsSubcollection || 'computations';
                 
                 // Determine category and computation name based on user type
+                // NOTE: ResultCommitter ignores category metadata if computation is in a non-core folder
+                // SignedInUserProfileMetrics is in popular-investor folder, so it's stored in popular-investor category
                 let category, computationName;
                 if (userType === 'POPULAR_INVESTOR') {
                     category = 'popular-investor';
                     computationName = 'PopularInvestorProfileMetrics';
                 } else {
-                    category = 'signed_in_user';
+                    // SignedInUserProfileMetrics is in popular-investor folder, so stored in popular-investor category
+                    category = 'popular-investor';
                     computationName = 'SignedInUserProfileMetrics';
                 }
                 
@@ -318,7 +326,8 @@ async function getUserSyncStatus(req, res, dependencies, config) {
             }
             
             // Check if request is stale (stuck in processing state for too long)
-            const STALE_THRESHOLD_MS = 5 * 60 * 1000; // 5 minutes
+            // Increased timeout to 15 minutes to account for computation time
+            const STALE_THRESHOLD_MS = 15 * 60 * 1000; // 15 minutes
             const processingStates = ['processing', 'dispatched', 'indexing', 'computing', 'queued'];
             const isProcessingState = processingStates.includes(status);
             
@@ -336,8 +345,22 @@ async function getUserSyncStatus(req, res, dependencies, config) {
                 const referenceTime = dispatchedAt || createdAt || updatedAt;
                 
                 if (referenceTime && (now - referenceTime) > STALE_THRESHOLD_MS) {
-                    isStale = true;
-                    logger.log('WARN', `[getUserSyncStatus] Detected stale request ${latestRequest.requestId} for user ${targetCidNum}. Status: ${status}, Age: ${Math.round((now - referenceTime) / 60000)} minutes`);
+                    // Before marking as stale, do one final check for computation results
+                    // Sometimes computation completes but status wasn't updated
+                    const finalCheck = await checkComputationResults(db, targetCidNum, userType, insightsCollection, resultsSub, compsSub, logger);
+                    if (!finalCheck) {
+                        isStale = true;
+                        logger.log('WARN', `[getUserSyncStatus] Detected stale request ${latestRequest.requestId} for user ${targetCidNum}. Status: ${status}, Age: ${Math.round((now - referenceTime) / 60000)} minutes`);
+                    } else {
+                        // Found results, update status to completed
+                        status = 'completed';
+                        await requestsSnapshot.docs[0].ref.update({
+                            status: 'completed',
+                            completedAt: FieldValue.serverTimestamp(),
+                            updatedAt: FieldValue.serverTimestamp()
+                        });
+                        logger.log('INFO', `[getUserSyncStatus] Found computation results on stale check, marked as completed`);
+                    }
                 }
             }
             
@@ -489,6 +512,80 @@ async function checkRateLimits(db, targetUserCid, logger) {
     };
 }
 
+/**
+ * Helper function to check for computation results
+ */
+async function checkComputationResults(db, targetCidNum, userType, insightsCollection, resultsSub, compsSub, logger) {
+    try {
+        // Determine category and computation name based on user type
+        // NOTE: ResultCommitter ignores category metadata if computation is in a non-core folder
+        // SignedInUserProfileMetrics is in popular-investor folder, so it's stored in popular-investor category
+        let category, computationName;
+        if (userType === 'POPULAR_INVESTOR') {
+            category = 'popular-investor';
+            computationName = 'PopularInvestorProfileMetrics';
+        } else {
+            // SignedInUserProfileMetrics is in popular-investor folder, so stored in popular-investor category
+            category = 'popular-investor';
+            computationName = 'SignedInUserProfileMetrics';
+        }
+        
+        // Check today and yesterday for computation results
+        const checkDate = new Date();
+        for (let i = 0; i < 2; i++) {
+            const dateStr = new Date(checkDate);
+            dateStr.setDate(checkDate.getDate() - i);
+            const dateStrFormatted = dateStr.toISOString().split('T')[0];
+            
+            const docRef = db.collection(insightsCollection)
+                .doc(dateStrFormatted)
+                .collection(resultsSub)
+                .doc(category)
+                .collection(compsSub)
+                .doc(computationName);
+            
+            const doc = await docRef.get();
+            if (doc.exists) {
+                const docData = doc.data();
+                let mergedData = null;
+                
+                // Check if data is sharded
+                if (docData._sharded === true && docData._shardCount) {
+                    const shardsCol = docRef.collection('_shards');
+                    const shardsSnapshot = await shardsCol.get();
+                    
+                    if (!shardsSnapshot.empty) {
+                        mergedData = {};
+                        for (const shardDoc of shardsSnapshot.docs) {
+                            const shardData = shardDoc.data();
+                            Object.assign(mergedData, shardData);
+                        }
+                    }
+                } else {
+                    const { tryDecompress } = require('./data_helpers');
+                    mergedData = tryDecompress(docData);
+                    
+                    if (typeof mergedData === 'string') {
+                        try {
+                            mergedData = JSON.parse(mergedData);
+                        } catch (e) {
+                            mergedData = null;
+                        }
+                    }
+                }
+                
+                if (mergedData && typeof mergedData === 'object' && mergedData[String(targetCidNum)]) {
+                    return true; // Found results
+                }
+            }
+        }
+        return false; // No results found
+    } catch (error) {
+        logger.log('WARN', `[checkComputationResults] Error checking computation results`, error);
+        return false;
+    }
+}
+
 module.exports = {
     requestUserSync,
     getUserSyncStatus

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -761,6 +761,26 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         
         logger.log('SUCCESS', `[On-Demand Update] Complete for ${username} (Portfolio: ${portfolioFetched ? '✓' : '✗'}, History: ${historyFetched ? '✓' : '✗'}, Social: ${socialFetched ? '✓' : '✗'})`);
         
+        // Send notification to requesting user if this is an on-demand sync
+        if (requestId && source === 'on_demand_sync' && metadata?.requestingUserCid) {
+            try {
+                const { notifyTaskEngineComplete } = require('../../generic-api/user-api/helpers/notification_helpers');
+                const success = portfolioFetched || historyFetched; // At least one should succeed
+                await notifyTaskEngineComplete(
+                    db,
+                    logger,
+                    metadata.requestingUserCid,
+                    requestId,
+                    username,
+                    success,
+                    success ? null : 'Failed to fetch portfolio or history data'
+                );
+            } catch (notifError) {
+                logger.log('WARN', `[On-Demand Update] Failed to send notification`, notifError);
+                // Non-critical, continue
+            }
+        }
+        
         // Update root data indexer after all data is fetched
         // Then trigger computations for both sync requests AND user signups
         const shouldTriggerComputations = (requestId && (source === 'on_demand_sync' || source === 'user_signup')) || isNewUser;
@@ -834,7 +854,9 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                                     requestId: requestId,
                                     userCid: cid,
                                     username: username,
-                                    targetCid: targetCid // Pass targetCid for optimization
+                                    targetCid: targetCid, // Pass targetCid for optimization
+                                    requestingUserCid: metadata?.requestingUserCid || null, // Pass for notifications
+                                    onDemand: true
                                 }
                             );
                             triggeredMessages.push(...messages);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.480",
+  "version": "1.0.491",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [