bulltrackers-module 1.0.492 → 1.0.494

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -345,7 +345,13 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
         };
     });
 
-    logger.log('WARN', `[Sweep] 🧹 Forcing ${tasksPayload.length} tasks to HIGH-MEM for ${date}.`);
+    const taskNames = tasksPayload.map(t => t.computation || t.name).join(', ');
+    logger.log('WARN', `[Sweep] 🧹 Forcing ${tasksPayload.length} tasks to HIGH-MEM for ${date}.`, {
+        date: date,
+        pass: passToRun,
+        tasks: tasksPayload.map(t => ({ name: t.computation || t.name, reason: 'sweep' })),
+        topic: config.computationTopicHighMem || 'computation-tasks-highmem'
+    });
 
     await pubsubUtils.batchPublishTasks(dependencies, {
         topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
@@ -503,6 +509,13 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
 
     const pubPromises = [];
     if (standardTasks.length > 0) {
+        const taskNames = standardTasks.map(t => t.computation || t.name).join(', ');
+        logger.log('INFO', `[Dispatcher] 📤 Dispatching ${standardTasks.length} standard tasks: ${taskNames}`, {
+            date: selectedDate,
+            pass: passToRun,
+            tasks: standardTasks.map(t => ({ name: t.computation || t.name, reason: t.triggerReason || 'new' })),
+            topic: config.computationTopicStandard || 'computation-tasks'
+        });
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicStandard || 'computation-tasks',
             tasks: standardTasks,
@@ -510,6 +523,13 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
         }));
     }
     if (highMemTasks.length > 0) {
+        const taskNames = highMemTasks.map(t => t.computation || t.name).join(', ');
+        logger.log('INFO', `[Dispatcher] 📤 Dispatching ${highMemTasks.length} high-memory tasks: ${taskNames}`, {
+            date: selectedDate,
+            pass: passToRun,
+            tasks: highMemTasks.map(t => ({ name: t.computation || t.name, reason: t.triggerReason || 'retry' })),
+            topic: config.computationTopicHighMem || 'computation-tasks-highmem'
+        });
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
             tasks: highMemTasks,

package/functions/computation-system/tools/FinalSweepReporter.js

@@ -164,7 +164,16 @@ class FinalSweepReporter {
 
         if (ledgerSnap.exists) {
             const data = ledgerSnap.data();
-            forensics.ledgerState = { status: data.status, workerId: data.workerId, error: data.error };
+            // Filter out undefined values to prevent Firestore errors
+            const ledgerState = {
+                status: data.status,
+                workerId: data.workerId
+            };
+            // Only include error if it's defined
+            if (data.error !== undefined && data.error !== null) {
+                ledgerState.error = data.error;
+            }
+            forensics.ledgerState = ledgerState;
 
             if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
                 const lastHb = data.telemetry?.lastHeartbeat ? new Date(data.telemetry.lastHeartbeat).getTime() : 0;

package/functions/computation-system/workflows/datafeederpipelineinstructions.md

@@ -1,32 +1,30 @@
-Below is a quick-reference guide for testing the **Data Feeder Pipeline** using the Google Cloud Console UI.
-
----
-
 # 🧪 Workflow Testing Guide
 
-When triggering a manual execution in the **GCP Workflows Console**, use the following JSON objects in the **Input** field to bypass the schedule and test specific components.
+Below is a quick-reference guide for manually triggering the **Data Feeder Pipeline** using the Google Cloud Console UI.
 
-### How to Run a Test
+## How to Run a Test
 
-1. Go to the **Workflows** page in your GCP Console.
-2. Click on `data-feeder-pipeline`.
+1. Navigate to **Workflows** in your GCP Console.
+2. Select `data-feeder-pipeline`.
 3. Click the **Execute** button at the top.
-4. Paste the relevant **JSON Input** from the table below into the input box.
+4. Paste the specific **JSON Input** from the table below into the input box to bypass schedules and target specific phases.
 
 ### Test Commands
 
-| Component to Test | JSON Input | Description |
-| --- | --- | --- |
-| **Market Data** | `{"target_step": "market"}` | Runs `price-fetcher`, `insights-fetcher`, and `market-data-indexer`. |
-| **Rankings** | `{"target_step": "rankings"}` | Runs the `fetch-popular-investors` function specifically. |
-| **Social Orchestrator** | `{"target_step": "social"}` | Runs the midnight social task and the midnight data indexer. |
-| **Global Sync** | `{"target_step": "global"}` | Triggers the `root-data-indexer` with no target date (Full Scan). |
-| **Full Pipeline** | `{}` | Runs the entire 24-hour cycle from the beginning (Standard Run). |
+| Phase to Test | JSON Input | Action & Description |
+| :--- | :--- | :--- |
+| **Market Data (Phase 1)** | `{"target_step": "market"}` | **Runs:** Price Fetcher & Insights Fetcher.<br>**Note:** Automatically triggers indexing after each fetch. Workflow will pause at "Wait for Midnight" upon completion. |
+| **Midnight Phase (Phase 3)** | `{"target_step": "midnight"}` | **Runs:** Popular Investor Rankings, Midnight Social Orchestrator, and Global Index Verification.<br>**Note:** Use this to test the critical 00:00 UTC logic without waiting for the daily schedule. |
+| **Social Loop (Phase 4)** | `{"target_step": "social"}` | **Runs:** Enters the recurring 3-hour social fetch loop.<br>**Warning:** Triggers `social_loop_start`, which begins with a 3-hour sleep (`wait_3_hours`) before the first execution. |
+| **Standard Run** | `{}` | **Runs:** The full 24-hour cycle starting from 22:00 UTC (Market Close). |
 
 ---
 
-### 💡 Pro-Tips for Testing
+### 💡 Testing Notes & Pro-Tips
 
-* **Variable Checking:** After an execution finishes, click the **Details** tab in the execution logs to see the final values of variables like `sleep_midnight` or `sleep_loop` to verify the UTC alignment math worked correctly.
-* **Permissions:** Ensure your user account or the service account running the workflow has the `roles/workflows.invoker` role to trigger these tests.
-* **Timeouts:** If testing the full loop, remember that the workflow will "pause" at the `sys.sleep` steps. You can see the status as **Active** while it waits for the next 3-hour window.
+* **Automatic Indexing:** As of V3.2, you will not see explicit "Index" steps in the workflow visualization for Price, Insights, or Rankings. These functions now trigger the `root-data-indexer` automatically upon completion. The only visible index step is the **Global Verification** in the Midnight Phase.
+* **Verification:** To confirm data was indexed during a test:
+    * Check the logs of the individual Cloud Functions (`price-fetcher`, etc.).
+    * Or, run the **Midnight Phase** test, which ends with the explicit `global_index_midnight` step.
+* **Variable Checking:** After execution, check the **Variables** tab to view `sleep_midnight` calculations to ensure UTC alignment is functioning correctly.
+* **Permissions:** Ensure the executor has `roles/workflows.invoker`.

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -5,6 +5,7 @@
 
 const { FieldValue } = require('@google-cloud/firestore');
 const crypto = require('crypto');
+const { tryDecompress } = require('./data_helpers');
 
 const RATE_LIMIT_HOURS = 1;
 const RATE_LIMIT_MS = RATE_LIMIT_HOURS * 60 * 60 * 1000;
@@ -253,7 +254,6 @@ async function getPiFetchStatus(req, res, dependencies, config) {
             
             const doc = await docRef.get();
             if (doc.exists) {
-                const { tryDecompress } = require('./data_helpers');
                 const data = tryDecompress(doc.data());
                 
                 if (data && data[String(piCidNum)]) {
@@ -316,7 +316,6 @@ async function getPiFetchStatus(req, res, dependencies, config) {
                             }
                         } else {
                             // Data is in the main document (compressed or not)
-                            const { tryDecompress } = require('./data_helpers');
                             mergedData = tryDecompress(docData);
                             
                             // Handle string decompression result

package/functions/generic-api/user-api/helpers/test_alert_helpers.js

@@ -0,0 +1,213 @@
+/**
+ * @fileoverview Test Alert Helpers
+ * Allows developers to send test alerts for testing the alert system
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+const { getAllAlertTypes, getAlertTypeByComputation } = require('../../../alert-system/helpers/alert_type_registry');
+const { isDeveloperAccount, getDevOverride } = require('./dev_helpers');
+
+/**
+ * POST /user/dev/test-alert
+ * Send a test alert to users
+ * 
+ * Request body:
+ * {
+ *   userCid: number (required) - Developer account CID
+ *   alertTypeId: string (optional) - Alert type ID, defaults to first available
+ *   targetUsers: 'all' | 'dev' | number[] (optional) - Who to send to, defaults to 'dev'
+ *   piCid: number (optional) - PI CID for the alert, defaults to 1
+ *   piUsername: string (optional) - PI username, defaults to 'TestPI'
+ *   metadata: object (optional) - Additional metadata for the alert
+ * }
+ */
+async function sendTestAlert(req, res, dependencies, config) {
+    const { db, logger } = dependencies;
+    const { userCid, alertTypeId, targetUsers = 'dev', piCid = 1, piUsername = 'TestPI', metadata = {} } = req.body;
+    
+    if (!userCid) {
+        return res.status(400).json({ error: "Missing userCid" });
+    }
+    
+    // SECURITY CHECK: Only allow developer accounts
+    if (!isDeveloperAccount(userCid)) {
+        logger.log('WARN', `[sendTestAlert] Unauthorized attempt by user ${userCid}`);
+        return res.status(403).json({ 
+            error: "Forbidden",
+            message: "Test alerts are only available for authorized developer accounts"
+        });
+    }
+    
+    try {
+        // Get alert type
+        let alertType;
+        if (alertTypeId) {
+            alertType = getAlertTypeByComputation(alertTypeId);
+            if (!alertType) {
+                const allTypes = getAllAlertTypes();
+                alertType = allTypes.find(t => t.id === alertTypeId);
+            }
+        }
+        
+        // Default to first available alert type if not specified
+        if (!alertType) {
+            const allTypes = getAllAlertTypes();
+            if (allTypes.length === 0) {
+                return res.status(400).json({ error: "No alert types available" });
+            }
+            alertType = allTypes[0];
+            logger.log('INFO', `[sendTestAlert] Using default alert type: ${alertType.id}`);
+        }
+        
+        // Determine target user CIDs
+        let targetCids = [];
+        
+        if (targetUsers === 'all') {
+            // Get all users from user_notifications collection
+            const notificationsSnapshot = await db.collection('user_notifications').get();
+            targetCids = notificationsSnapshot.docs.map(doc => Number(doc.id));
+            logger.log('INFO', `[sendTestAlert] Sending to all ${targetCids.length} users`);
+        } else if (targetUsers === 'dev') {
+            // Get all developer accounts with dev override enabled
+            const devOverridesCollection = config.devOverridesCollection || 'dev_overrides';
+            const devOverridesSnapshot = await db.collection(devOverridesCollection).get();
+            
+            for (const doc of devOverridesSnapshot.docs) {
+                const data = doc.data();
+                if (data.enabled === true) {
+                    targetCids.push(Number(doc.id));
+                }
+            }
+            
+            // Also include the requesting developer
+            if (!targetCids.includes(Number(userCid))) {
+                targetCids.push(Number(userCid));
+            }
+            
+            logger.log('INFO', `[sendTestAlert] Sending to ${targetCids.length} developer accounts`);
+        } else if (Array.isArray(targetUsers)) {
+            // Specific user CIDs
+            targetCids = targetUsers.map(cid => Number(cid)).filter(cid => !isNaN(cid) && cid > 0);
+            logger.log('INFO', `[sendTestAlert] Sending to ${targetCids.length} specific users`);
+        } else {
+            return res.status(400).json({ 
+                error: "Invalid targetUsers",
+                message: "targetUsers must be 'all', 'dev', or an array of user CIDs"
+            });
+        }
+        
+        if (targetCids.length === 0) {
+            return res.status(400).json({ 
+                error: "No target users",
+                message: "No users found matching the target criteria"
+            });
+        }
+        
+        // Generate alert message
+        const { generateAlertMessage } = require('../../../alert-system/helpers/alert_type_registry');
+        const alertMessage = generateAlertMessage(alertType, piUsername, {
+            ...metadata,
+            isTest: true,
+            testSentBy: Number(userCid),
+            testSentAt: new Date().toISOString()
+        });
+        
+        // Create notifications for each target user
+        const batch = db.batch();
+        const notificationRefs = [];
+        const counterUpdates = {};
+        const today = new Date().toISOString().split('T')[0];
+        
+        for (const targetCid of targetCids) {
+            const notificationId = `test_alert_${Date.now()}_${targetCid}_${piCid}_${Math.random().toString(36).substring(2, 9)}`;
+            const notificationRef = db.collection('user_notifications')
+                .doc(String(targetCid))
+                .collection('notifications')
+                .doc(notificationId);
+            
+            const notificationData = {
+                id: notificationId,
+                type: 'alert',
+                title: `[TEST] ${alertType.name}`,
+                message: alertMessage,
+                read: false,
+                createdAt: FieldValue.serverTimestamp(),
+                metadata: {
+                    piCid: Number(piCid),
+                    piUsername: piUsername,
+                    alertType: alertType.id,
+                    alertTypeName: alertType.name,
+                    computationName: alertType.computationName,
+                    computationDate: today,
+                    severity: alertType.severity,
+                    isTest: true,
+                    testSentBy: Number(userCid),
+                    testSentAt: new Date().toISOString(),
+                    ...metadata
+                }
+            };
+            
+            batch.set(notificationRef, notificationData);
+            notificationRefs.push(notificationRef);
+            
+            // Track counter updates
+            if (!counterUpdates[targetCid]) {
+                counterUpdates[targetCid] = {
+                    date: today,
+                    unreadCount: 0,
+                    totalCount: 0,
+                    byType: {}
+                };
+            }
+            counterUpdates[targetCid].unreadCount += 1;
+            counterUpdates[targetCid].totalCount += 1;
+            counterUpdates[targetCid].byType[alertType.id] = 
+                (counterUpdates[targetCid].byType[alertType.id] || 0) + 1;
+        }
+        
+        // Update notification counters
+        for (const [targetCid, counter] of Object.entries(counterUpdates)) {
+            const counterRef = db.collection('user_notifications')
+                .doc(String(targetCid))
+                .collection('counters')
+                .doc(counter.date);
+            
+            batch.set(counterRef, {
+                date: counter.date,
+                unreadCount: FieldValue.increment(counter.unreadCount),
+                totalCount: FieldValue.increment(counter.totalCount),
+                [`byType.${alertType.id}`]: FieldValue.increment(counter.byType[alertType.id] || 0),
+                lastUpdated: FieldValue.serverTimestamp()
+            }, { merge: true });
+        }
+        
+        // Commit batch
+        await batch.commit();
+        
+        logger.log('SUCCESS', `[sendTestAlert] Created ${notificationRefs.length} test notifications for alert type ${alertType.id}`);
+        
+        return res.status(200).json({
+            success: true,
+            message: `Test alert sent to ${targetCids.length} users`,
+            alertType: {
+                id: alertType.id,
+                name: alertType.name,
+                computationName: alertType.computationName
+            },
+            targetUsers: {
+                count: targetCids.length,
+                cids: targetCids
+            },
+            piCid: Number(piCid),
+            piUsername: piUsername,
+            notificationsCreated: notificationRefs.length
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[sendTestAlert] Error sending test alert:`, error);
+        return res.status(500).json({ error: error.message });
+    }
+}
+
+module.exports = { sendTestAlert };
+

package/functions/generic-api/user-api/index.js

@@ -12,6 +12,7 @@ const { setDevOverride, getDevOverrideStatus } = require('./helpers/dev_helpers'
 const { getAlertTypes, getDynamicWatchlistComputations, getUserAlerts, getAlertCount, markAlertRead, markAllAlertsRead, deleteAlert } = require('./helpers/alert_helpers');
 const { requestPiFetch, getPiFetchStatus } = require('./helpers/on_demand_fetch_helpers');
 const { requestUserSync, getUserSyncStatus } = require('./helpers/user_sync_helpers');
+const { sendTestAlert } = require('./helpers/test_alert_helpers');
 
 module.exports = (dependencies, config) => {
     const router = express.Router();
@@ -87,6 +88,7 @@ module.exports = (dependencies, config) => {
     // --- Developer Mode (only for whitelisted developer accounts) ---
     router.post('/dev/override', (req, res) => setDevOverride(req, res, dependencies, config));
     router.get('/dev/override', (req, res) => getDevOverrideStatus(req, res, dependencies, config));
+    router.post('/dev/test-alert', (req, res) => sendTestAlert(req, res, dependencies, config));
     
     // --- Alert Management ---
     router.get('/me/alert-types', (req, res) => getAlertTypes(req, res, dependencies, config));

package/functions/root-data-indexer/index.js

@@ -270,19 +270,33 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             ]);
             
             // Check if date exists in tracking documents
+            // The _dates document uses dot notation: fetchedDates.2025-12-29: true
+            // When read, this becomes: { fetchedDates: { "2025-12-29": true } }
             let foundPISocial = false;
             let foundSignedInSocial = false;
             
             if (piSocialTrackingDoc.exists) {
                 const data = piSocialTrackingDoc.data();
-                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                // Check both nested structure and flat dot-notation structure
+                if (data.fetchedDates && typeof data.fetchedDates === 'object') {
+                    if (data.fetchedDates[dateStr] === true) {
+                        foundPISocial = true;
+                    }
+                } else if (data[`fetchedDates.${dateStr}`] === true) {
+                    // Handle flat dot-notation structure (if Firestore stores it that way)
                     foundPISocial = true;
                 }
             }
             
             if (signedInSocialTrackingDoc.exists) {
                 const data = signedInSocialTrackingDoc.data();
-                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                // Check both nested structure and flat dot-notation structure
+                if (data.fetchedDates && typeof data.fetchedDates === 'object') {
+                    if (data.fetchedDates[dateStr] === true) {
+                        foundSignedInSocial = true;
+                    }
+                } else if (data[`fetchedDates.${dateStr}`] === true) {
+                    // Handle flat dot-notation structure (if Firestore stores it that way)
                     foundSignedInSocial = true;
                 }
             } 

package/functions/task-engine/handler_creator.js

@@ -235,12 +235,27 @@ async function handleRequest(message, context, configObj, dependencies) {
                 await handlePopularInvestorUpdate(taskData, configObj, dependencies);
                 break;
             case 'ON_DEMAND_USER_UPDATE':
-                const onDemandData = data || payload;
-                if (!onDemandData.cid || !onDemandData.username) {
-                    logger.log('ERROR', `[TaskEngine] ON_DEMAND_USER_UPDATE missing required fields`, { data: onDemandData });
+                // For ON_DEMAND_USER_UPDATE, the entire payload IS the task data
+                // (not wrapped in a 'data' field like other task types)
+                // Extract task data from payload, excluding 'type'
+                const onDemandTaskData = data || {
+                    cid: payload.cid,
+                    username: payload.username,
+                    requestId: payload.requestId,
+                    source: payload.source,
+                    requestedBy: payload.requestedBy,
+                    effectiveRequestedBy: payload.effectiveRequestedBy,
+                    metadata: payload.metadata,
+                    priority: payload.priority,
+                    data: payload.data // Include nested data object (includeSocial, since, etc.)
+                };
+                
+                if (!onDemandTaskData.cid || !onDemandTaskData.username) {
+                    logger.log('ERROR', `[TaskEngine] ON_DEMAND_USER_UPDATE missing required fields (cid or username)`, { payload, onDemandTaskData });
                     return;
                 }
-                await handleOnDemandUserUpdate(onDemandData, configObj, dependencies);
+                
+                await handleOnDemandUserUpdate(onDemandTaskData, configObj, dependencies);
                 break;
             case 'SOCIAL_INSTRUMENT_FETCH':
             case 'SOCIAL_PI_FETCH':

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -500,6 +500,9 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     // [FIX] Destructure dependencies first
     const { logger, proxyManager, batchManager, headerManager, db } = dependencies;
     
+    // Import notification helper once at the top
+    const { notifyTaskEngineComplete } = require('../../generic-api/user-api/helpers/notification_helpers');
+    
     // Validate and set API URLs with defaults and fallbacks
     const ETORO_API_PORTFOLIO_URL = config.ETORO_API_PORTFOLIO_URL || process.env.ETORO_API_PORTFOLIO_URL || 'https://www.etoro.com/sapi/portfolios/portfolio';
     const ETORO_API_HISTORY_URL = config.ETORO_API_HISTORY_URL || process.env.ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
@@ -558,6 +561,48 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     let fetchSuccess = false;
     let portfolioFetched = false;
     let historyFetched = false;
+    let criticalError = null; // Track critical errors that should fail the sync
+
+    // Helper function to mark sync as failed and send notification
+    const markSyncFailed = async (errorMessage, stage) => {
+        if (requestId && source === 'on_demand_sync' && db) {
+            try {
+                const requestRef = db.collection('user_sync_requests')
+                    .doc(String(cid))
+                    .collection('requests')
+                    .doc(requestId);
+                
+                await requestRef.update({
+                    status: 'failed',
+                    error: errorMessage,
+                    failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    failedStage: stage
+                });
+                
+                // Send error notification
+                if (metadata?.requestingUserCid) {
+                    try {
+                        await notifyTaskEngineComplete(
+                            db,
+                            logger,
+                            metadata.requestingUserCid,
+                            requestId,
+                            username,
+                            false,
+                            errorMessage
+                        );
+                    } catch (notifErr) {
+                        logger.log('WARN', `[On-Demand Update] Failed to send error notification`, notifErr);
+                    }
+                }
+                
+                logger.log('ERROR', `[On-Demand Update] Marked sync ${requestId} as failed at stage: ${stage}. Error: ${errorMessage}`);
+            } catch (updateErr) {
+                logger.log('ERROR', `[On-Demand Update] Failed to mark sync as failed for ${requestId}`, updateErr);
+            }
+        }
+    };
 
     try {
         // Portfolio Fetch (only if portfolioOnly is true)
@@ -597,7 +642,10 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     portfolioSuccess = true;
                 } catch (fetchErr) {
                     logger.log('ERROR', `[On-Demand] Direct fetch also failed for ${username}`, fetchErr);
-                    throw new Error(`Failed to fetch portfolio for ${cid}. Direct: ${fetchErr.message}`);
+                    const errorMsg = `Failed to fetch portfolio for ${cid}. Direct: ${fetchErr.message}`;
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'portfolio_fetch');
+                    throw criticalError;
                 }
             }
 
@@ -608,7 +656,15 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
             fetchSuccess = true;
             portfolioFetched = true;
 
-            await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
+            try {
+                await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
+            } catch (batchErr) {
+                const errorMsg = `Failed to store portfolio data: ${batchErr.message}`;
+                logger.log('ERROR', `[On-Demand] ${errorMsg}`, batchErr);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'portfolio_storage');
+                throw criticalError;
+            }
         } else {
             logger.log('INFO', `[On-Demand Update] Skipping portfolio fetch (portfolioOnly=false) for ${username}`);
         }
@@ -660,23 +716,49 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         }
         
         if (historySuccess && historyRes.ok) {
-            const historyData = await historyRes.json();
-            historyData.fetchedAt = new Date();
-            
-            const VALID_REASONS = [0, 1, 5]; 
-            if (historyData.PublicHistoryPositions) {
-                historyData.PublicHistoryPositions = historyData.PublicHistoryPositions.filter(
-                    p => VALID_REASONS.includes(p.CloseReason)
-                );
+            try {
+                const historyData = await historyRes.json();
+                historyData.fetchedAt = new Date();
+                
+                const VALID_REASONS = [0, 1, 5]; 
+                if (historyData.PublicHistoryPositions) {
+                    historyData.PublicHistoryPositions = historyData.PublicHistoryPositions.filter(
+                        p => VALID_REASONS.includes(p.CloseReason)
+                    );
+                }
+                
+                try {
+                    await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
+                    historyFetched = true;
+                } catch (batchErr) {
+                    const errorMsg = `Failed to store history data: ${batchErr.message}`;
+                    logger.log('ERROR', `[On-Demand] ${errorMsg}`, batchErr);
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'history_storage');
+                    throw criticalError;
+                }
+            } catch (parseErr) {
+                const errorMsg = `Failed to parse history data: ${parseErr.message}`;
+                logger.log('ERROR', `[On-Demand] ${errorMsg}`, parseErr);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'history_parse');
+                throw criticalError;
             }
-                await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
-                historyFetched = true;
+        } else {
+            // History fetch failed - this is critical if portfolioOnly is true
+            if (portfolioOnly) {
+                const errorMsg = `History fetch failed for ${username} (status: ${historyRes?.status || 'unknown'})`;
+                logger.log('ERROR', `[On-Demand Update] ${errorMsg}`);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'history_fetch');
+                throw criticalError;
             } else {
-                logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes.status})`);
+                logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes?.status || 'unknown'})`);
             }
-        } else {
-            logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
         }
+    } else {
+        logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
+    }
 
         // Fetch social data if requested (for user signup or explicit request)
         let socialFetched = false;
@@ -764,7 +846,6 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         // Send notification to requesting user if this is an on-demand sync
         if (requestId && source === 'on_demand_sync' && metadata?.requestingUserCid) {
             try {
-                const { notifyTaskEngineComplete } = require('../../generic-api/user-api/helpers/notification_helpers');
                 const success = portfolioFetched || historyFetched; // At least one should succeed
                 await notifyTaskEngineComplete(
                     db,
@@ -816,14 +897,12 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                         updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
                     });
                 } catch (indexerError) {
-                    logger.log('ERROR', `[On-Demand Update] Failed to run root data indexer for ${today}`, indexerError);
-                    // Continue anyway - computation might still work if index already exists
-                    // But update status to indicate we tried
-                    await requestRef.update({
-                        status: 'computing',
-                        indexerError: indexerError.message,
-                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-                    });
+                    const errorMsg = `Failed to run root data indexer: ${indexerError.message}`;
+                    logger.log('ERROR', `[On-Demand Update] ${errorMsg}`, indexerError);
+                    // Root data indexing is critical - if it fails, computations won't know data exists
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'root_data_indexing');
+                    throw criticalError;
                 }
                 
                 // Trigger computations with dependency chain resolution
@@ -870,22 +949,30 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     // Don't mark as completed yet - wait for computation to finish
                     // The status will remain 'computing' until computation completes
                 } else {
-                    logger.log('WARN', `[On-Demand Update] PubSub not available, cannot trigger computation`);
-                    // Mark as completed if we can't trigger computation
-                    await requestRef.update({
-                        status: 'completed',
-                        completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
-                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-                    });
+                    const errorMsg = 'PubSub not available, cannot trigger computation';
+                    logger.log('ERROR', `[On-Demand Update] ${errorMsg}`);
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'computation_trigger');
+                    throw criticalError;
                 }
             } catch (err) {
-                logger.log('WARN', `[On-Demand Update] Failed to update request status or trigger computation for ${requestId}`, err);
+                // If this is a critical error we already handled, re-throw it
+                if (criticalError) {
+                    throw criticalError;
+                }
+                // Otherwise, this is an unexpected error in the computation triggering block
+                const errorMsg = `Failed to trigger computations: ${err.message}`;
+                logger.log('ERROR', `[On-Demand Update] ${errorMsg}`, err);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'computation_trigger');
+                throw criticalError;
             }
         }
     } catch (error) {
         logger.log('ERROR', `[On-Demand Update] Failed for ${username}`, error);
         
         // Update request status to failed if this is a sync request
+        // Only update if we haven't already marked it as failed (to avoid duplicate updates)
         if (requestId && source === 'on_demand_sync' && db) {
             try {
                 const requestRef = db.collection('user_sync_requests')
@@ -893,14 +980,39 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     .collection('requests')
                     .doc(requestId);
                 
-                const errorMessage = error.message || 'Unknown error occurred';
-                await requestRef.update({
-                    status: 'failed',
-                    error: errorMessage,
-                    failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
-                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-                });
-                logger.log('INFO', `[On-Demand Update] Updated sync request ${requestId} to failed: ${errorMessage}`);
+                // Check current status to avoid overwriting if already marked as failed
+                const currentRequest = await requestRef.get();
+                if (currentRequest.exists && currentRequest.data().status !== 'failed') {
+                    const errorMessage = error.message || 'Unknown error occurred';
+                    const failedStage = error.failedStage || 'unknown';
+                    
+                    await requestRef.update({
+                        status: 'failed',
+                        error: errorMessage,
+                        failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        failedStage: failedStage
+                    });
+                    
+                    // Send error notification if not already sent
+                    if (metadata?.requestingUserCid) {
+                        try {
+                            await notifyTaskEngineComplete(
+                                db,
+                                logger,
+                                metadata.requestingUserCid,
+                                requestId,
+                                username,
+                                false,
+                                errorMessage
+                            );
+                        } catch (notifErr) {
+                            logger.log('WARN', `[On-Demand Update] Failed to send error notification`, notifErr);
+                        }
+                    }
+                    
+                    logger.log('INFO', `[On-Demand Update] Updated sync request ${requestId} to failed: ${errorMessage} (stage: ${failedStage})`);
+                }
             } catch (err) {
                 logger.log('WARN', `[On-Demand Update] Failed to update request status to failed for ${requestId}`, err);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.492",
+  "version": "1.0.494",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [