bulltrackers-module 1.0.493 → 1.0.495

package/functions/alert-system/helpers/alert_helpers.js

@@ -29,15 +29,41 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
     // 3. Generate alert message
     const alertMessage = generateAlertMessage(alertType, piUsername, computationMetadata);
     
+    // Helper function to get Firebase UID from eToro CID
+    async function getFirebaseUidFromCid(etoroCid) {
+      const signedInUsersSnapshot = await db.collection('signedInUsers')
+        .where('etoroCID', '==', Number(etoroCid))
+        .limit(1)
+        .get();
+      
+      if (!signedInUsersSnapshot.empty) {
+        return signedInUsersSnapshot.docs[0].id; // Firebase UID is the document ID
+      }
+      return null;
+    }
+    
     // 4. Create notifications for each subscribed user (using user_notifications collection)
+    // Convert eToro CIDs to Firebase UIDs first
     const batch = db.batch();
     const notificationRefs = [];
     const counterUpdates = {};
+    const uidMapping = {}; // Cache for CID -> UID mappings
     
     for (const subscription of subscriptions) {
+      // Get Firebase UID for this user's eToro CID
+      let firebaseUid = uidMapping[subscription.userCid];
+      if (!firebaseUid) {
+        firebaseUid = await getFirebaseUidFromCid(subscription.userCid);
+        if (!firebaseUid) {
+          logger.log('WARN', `[processAlertForPI] Could not find Firebase UID for user CID ${subscription.userCid}, skipping notification`);
+          continue; // Skip this user if we can't find their Firebase UID
+        }
+        uidMapping[subscription.userCid] = firebaseUid;
+      }
+      
       const notificationId = `alert_${Date.now()}_${subscription.userCid}_${piCid}_${Math.random().toString(36).substring(2, 9)}`;
       const notificationRef = db.collection('user_notifications')
-        .doc(String(subscription.userCid))
+        .doc(firebaseUid) // Use Firebase UID, not eToro CID
         .collection('notifications')
         .doc(notificationId);
       
@@ -48,6 +74,7 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
         message: alertMessage,
         read: false,
         createdAt: FieldValue.serverTimestamp(),
+        timestamp: FieldValue.serverTimestamp(), // Also include timestamp for ordering
         metadata: {
           piCid: Number(piCid),
           piUsername: piUsername,
@@ -65,26 +92,26 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
       batch.set(notificationRef, notificationData);
       notificationRefs.push(notificationRef);
       
-      // Track counter updates
+      // Track counter updates (using Firebase UID as key)
       const dateKey = computationDate || new Date().toISOString().split('T')[0];
-      if (!counterUpdates[subscription.userCid]) {
-        counterUpdates[subscription.userCid] = {
+      if (!counterUpdates[firebaseUid]) {
+        counterUpdates[firebaseUid] = {
           date: dateKey,
           unreadCount: 0,
           totalCount: 0,
           byType: {}
         };
       }
-      counterUpdates[subscription.userCid].unreadCount += 1;
-      counterUpdates[subscription.userCid].totalCount += 1;
-      counterUpdates[subscription.userCid].byType[alertType.id] = 
-        (counterUpdates[subscription.userCid].byType[alertType.id] || 0) + 1;
+      counterUpdates[firebaseUid].unreadCount += 1;
+      counterUpdates[firebaseUid].totalCount += 1;
+      counterUpdates[firebaseUid].byType[alertType.id] = 
+        (counterUpdates[firebaseUid].byType[alertType.id] || 0) + 1;
     }
     
-    // 5. Update notification counters
-    for (const [userCid, counter] of Object.entries(counterUpdates)) {
+    // 5. Update notification counters (using Firebase UIDs)
+    for (const [firebaseUid, counter] of Object.entries(counterUpdates)) {
       const counterRef = db.collection('user_notifications')
-        .doc(String(userCid))
+        .doc(firebaseUid) // Use Firebase UID, not eToro CID
         .collection('counters')
         .doc(counter.date);
       

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -345,7 +345,13 @@ async function handleSweepDispatch(config, dependencies, computationManifest, re
         };
     });
 
-    logger.log('WARN', `[Sweep] 🧹 Forcing ${tasksPayload.length} tasks to HIGH-MEM for ${date}.`);
+    const taskNames = tasksPayload.map(t => t.computation || t.name).join(', ');
+    logger.log('WARN', `[Sweep] 🧹 Forcing ${tasksPayload.length} tasks to HIGH-MEM for ${date}.`, {
+        date: date,
+        pass: passToRun,
+        tasks: tasksPayload.map(t => ({ name: t.computation || t.name, reason: 'sweep' })),
+        topic: config.computationTopicHighMem || 'computation-tasks-highmem'
+    });
 
     await pubsubUtils.batchPublishTasks(dependencies, {
         topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
@@ -503,6 +509,13 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
 
     const pubPromises = [];
     if (standardTasks.length > 0) {
+        const taskNames = standardTasks.map(t => t.computation || t.name).join(', ');
+        logger.log('INFO', `[Dispatcher] 📤 Dispatching ${standardTasks.length} standard tasks: ${taskNames}`, {
+            date: selectedDate,
+            pass: passToRun,
+            tasks: standardTasks.map(t => ({ name: t.computation || t.name, reason: t.triggerReason || 'new' })),
+            topic: config.computationTopicStandard || 'computation-tasks'
+        });
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicStandard || 'computation-tasks',
             tasks: standardTasks,
@@ -510,6 +523,13 @@ async function handleStandardDispatch(config, dependencies, computationManifest,
         }));
     }
     if (highMemTasks.length > 0) {
+        const taskNames = highMemTasks.map(t => t.computation || t.name).join(', ');
+        logger.log('INFO', `[Dispatcher] 📤 Dispatching ${highMemTasks.length} high-memory tasks: ${taskNames}`, {
+            date: selectedDate,
+            pass: passToRun,
+            tasks: highMemTasks.map(t => ({ name: t.computation || t.name, reason: t.triggerReason || 'retry' })),
+            topic: config.computationTopicHighMem || 'computation-tasks-highmem'
+        });
         pubPromises.push(pubsubUtils.batchPublishTasks(dependencies, {
             topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
             tasks: highMemTasks,

package/functions/computation-system/tools/FinalSweepReporter.js

@@ -164,7 +164,16 @@ class FinalSweepReporter {
 
         if (ledgerSnap.exists) {
             const data = ledgerSnap.data();
-            forensics.ledgerState = { status: data.status, workerId: data.workerId, error: data.error };
+            // Filter out undefined values to prevent Firestore errors
+            const ledgerState = {
+                status: data.status,
+                workerId: data.workerId
+            };
+            // Only include error if it's defined
+            if (data.error !== undefined && data.error !== null) {
+                ledgerState.error = data.error;
+            }
+            forensics.ledgerState = ledgerState;
 
             if (['PENDING', 'IN_PROGRESS'].includes(data.status)) {
                 const lastHb = data.telemetry?.lastHeartbeat ? new Date(data.telemetry.lastHeartbeat).getTime() : 0;

package/functions/computation-system/workflows/datafeederpipelineinstructions.md

@@ -1,32 +1,30 @@
-Below is a quick-reference guide for testing the **Data Feeder Pipeline** using the Google Cloud Console UI.
-
----
-
 # 🧪 Workflow Testing Guide
 
-When triggering a manual execution in the **GCP Workflows Console**, use the following JSON objects in the **Input** field to bypass the schedule and test specific components.
+Below is a quick-reference guide for manually triggering the **Data Feeder Pipeline** using the Google Cloud Console UI.
 
-### How to Run a Test
+## How to Run a Test
 
-1. Go to the **Workflows** page in your GCP Console.
-2. Click on `data-feeder-pipeline`.
+1. Navigate to **Workflows** in your GCP Console.
+2. Select `data-feeder-pipeline`.
 3. Click the **Execute** button at the top.
-4. Paste the relevant **JSON Input** from the table below into the input box.
+4. Paste the specific **JSON Input** from the table below into the input box to bypass schedules and target specific phases.
 
 ### Test Commands
 
-| Component to Test | JSON Input | Description |
-| --- | --- | --- |
-| **Market Data** | `{"target_step": "market"}` | Runs `price-fetcher`, `insights-fetcher`, and `market-data-indexer`. |
-| **Rankings** | `{"target_step": "rankings"}` | Runs the `fetch-popular-investors` function specifically. |
-| **Social Orchestrator** | `{"target_step": "social"}` | Runs the midnight social task and the midnight data indexer. |
-| **Global Sync** | `{"target_step": "global"}` | Triggers the `root-data-indexer` with no target date (Full Scan). |
-| **Full Pipeline** | `{}` | Runs the entire 24-hour cycle from the beginning (Standard Run). |
+| Phase to Test | JSON Input | Action & Description |
+| :--- | :--- | :--- |
+| **Market Data (Phase 1)** | `{"target_step": "market"}` | **Runs:** Price Fetcher & Insights Fetcher.<br>**Note:** Automatically triggers indexing after each fetch. Workflow will pause at "Wait for Midnight" upon completion. |
+| **Midnight Phase (Phase 3)** | `{"target_step": "midnight"}` | **Runs:** Popular Investor Rankings, Midnight Social Orchestrator, and Global Index Verification.<br>**Note:** Use this to test the critical 00:00 UTC logic without waiting for the daily schedule. |
+| **Social Loop (Phase 4)** | `{"target_step": "social"}` | **Runs:** Enters the recurring 3-hour social fetch loop.<br>**Warning:** Triggers `social_loop_start`, which begins with a 3-hour sleep (`wait_3_hours`) before the first execution. |
+| **Standard Run** | `{}` | **Runs:** The full 24-hour cycle starting from 22:00 UTC (Market Close). |
 
 ---
 
-### 💡 Pro-Tips for Testing
+### 💡 Testing Notes & Pro-Tips
 
-* **Variable Checking:** After an execution finishes, click the **Details** tab in the execution logs to see the final values of variables like `sleep_midnight` or `sleep_loop` to verify the UTC alignment math worked correctly.
-* **Permissions:** Ensure your user account or the service account running the workflow has the `roles/workflows.invoker` role to trigger these tests.
-* **Timeouts:** If testing the full loop, remember that the workflow will "pause" at the `sys.sleep` steps. You can see the status as **Active** while it waits for the next 3-hour window.
+* **Automatic Indexing:** As of V3.2, you will not see explicit "Index" steps in the workflow visualization for Price, Insights, or Rankings. These functions now trigger the `root-data-indexer` automatically upon completion. The only visible index step is the **Global Verification** in the Midnight Phase.
+* **Verification:** To confirm data was indexed during a test:
+    * Check the logs of the individual Cloud Functions (`price-fetcher`, etc.).
+    * Or, run the **Midnight Phase** test, which ends with the explicit `global_index_midnight` step.
+* **Variable Checking:** After execution, check the **Variables** tab to view `sleep_midnight` calculations to ensure UTC alignment is functioning correctly.
+* **Permissions:** Ensure the executor has `roles/workflows.invoker`.

package/functions/generic-api/user-api/helpers/test_alert_helpers.js

@@ -0,0 +1,251 @@
+/**
+ * @fileoverview Test Alert Helpers
+ * Allows developers to send test alerts for testing the alert system
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+const { getAllAlertTypes, getAlertTypeByComputation } = require('../../../alert-system/helpers/alert_type_registry');
+const { isDeveloperAccount, getDevOverride } = require('./dev_helpers');
+
+/**
+ * POST /user/dev/test-alert
+ * Send a test alert to users
+ * 
+ * Request body:
+ * {
+ *   userCid: number (required) - Developer account CID
+ *   alertTypeId: string (optional) - Alert type ID, defaults to first available
+ *   targetUsers: 'all' | 'dev' | number[] (optional) - Who to send to, defaults to 'dev'
+ *   piCid: number (optional) - PI CID for the alert, defaults to 1
+ *   piUsername: string (optional) - PI username, defaults to 'TestPI'
+ *   metadata: object (optional) - Additional metadata for the alert
+ * }
+ */
+async function sendTestAlert(req, res, dependencies, config) {
+    const { db, logger } = dependencies;
+    const { userCid, alertTypeId, targetUsers = 'dev', piCid = 1, piUsername = 'TestPI', metadata = {} } = req.body;
+    
+    if (!userCid) {
+        return res.status(400).json({ error: "Missing userCid" });
+    }
+    
+    // SECURITY CHECK: Only allow developer accounts
+    if (!isDeveloperAccount(userCid)) {
+        logger.log('WARN', `[sendTestAlert] Unauthorized attempt by user ${userCid}`);
+        return res.status(403).json({ 
+            error: "Forbidden",
+            message: "Test alerts are only available for authorized developer accounts"
+        });
+    }
+    
+    try {
+        // Get alert type
+        let alertType;
+        if (alertTypeId) {
+            alertType = getAlertTypeByComputation(alertTypeId);
+            if (!alertType) {
+                const allTypes = getAllAlertTypes();
+                alertType = allTypes.find(t => t.id === alertTypeId);
+            }
+        }
+        
+        // Default to first available alert type if not specified
+        if (!alertType) {
+            const allTypes = getAllAlertTypes();
+            if (allTypes.length === 0) {
+                return res.status(400).json({ error: "No alert types available" });
+            }
+            alertType = allTypes[0];
+            logger.log('INFO', `[sendTestAlert] Using default alert type: ${alertType.id}`);
+        }
+        
+        // Helper function to get Firebase UID from eToro CID
+        async function getFirebaseUidFromCid(etoroCid) {
+            const signedInUsersSnapshot = await db.collection('signedInUsers')
+                .where('etoroCID', '==', Number(etoroCid))
+                .limit(1)
+                .get();
+            
+            if (!signedInUsersSnapshot.empty) {
+                return signedInUsersSnapshot.docs[0].id; // Firebase UID is the document ID
+            }
+            return null;
+        }
+        
+        // Determine target users (as Firebase UIDs)
+        let targetFirebaseUids = [];
+        
+        if (targetUsers === 'all') {
+            // Get all users from signedInUsers collection (who have etoroCID)
+            const signedInUsersSnapshot = await db.collection('signedInUsers')
+                .where('etoroCID', '!=', null)
+                .get();
+            
+            targetFirebaseUids = signedInUsersSnapshot.docs.map(doc => doc.id);
+            logger.log('INFO', `[sendTestAlert] Sending to all ${targetFirebaseUids.length} users`);
+        } else if (targetUsers === 'dev') {
+            // Get all developer accounts with dev override enabled
+            const devOverridesCollection = config.devOverridesCollection || 'dev_overrides';
+            const devOverridesSnapshot = await db.collection(devOverridesCollection).get();
+            
+            const devCids = [];
+            for (const doc of devOverridesSnapshot.docs) {
+                const data = doc.data();
+                if (data.enabled === true) {
+                    devCids.push(Number(doc.id));
+                }
+            }
+            
+            // Also include the requesting developer
+            if (!devCids.includes(Number(userCid))) {
+                devCids.push(Number(userCid));
+            }
+            
+            // Convert eToro CIDs to Firebase UIDs
+            for (const cid of devCids) {
+                const firebaseUid = await getFirebaseUidFromCid(cid);
+                if (firebaseUid) {
+                    targetFirebaseUids.push(firebaseUid);
+                } else {
+                    logger.log('WARN', `[sendTestAlert] Could not find Firebase UID for developer CID ${cid}`);
+                }
+            }
+            
+            logger.log('INFO', `[sendTestAlert] Sending to ${targetFirebaseUids.length} developer accounts`);
+        } else if (Array.isArray(targetUsers)) {
+            // Specific user CIDs - convert to Firebase UIDs
+            const specificCids = targetUsers.map(cid => Number(cid)).filter(cid => !isNaN(cid) && cid > 0);
+            
+            for (const cid of specificCids) {
+                const firebaseUid = await getFirebaseUidFromCid(cid);
+                if (firebaseUid) {
+                    targetFirebaseUids.push(firebaseUid);
+                } else {
+                    logger.log('WARN', `[sendTestAlert] Could not find Firebase UID for CID ${cid}`);
+                }
+            }
+            
+            logger.log('INFO', `[sendTestAlert] Sending to ${targetFirebaseUids.length} specific users (from ${specificCids.length} CIDs)`);
+        } else {
+            return res.status(400).json({ 
+                error: "Invalid targetUsers",
+                message: "targetUsers must be 'all', 'dev', or an array of user CIDs"
+            });
+        }
+        
+        if (targetFirebaseUids.length === 0) {
+            return res.status(400).json({ 
+                error: "No target users",
+                message: "No users found matching the target criteria"
+            });
+        }
+        
+        // Generate alert message
+        const { generateAlertMessage } = require('../../../alert-system/helpers/alert_type_registry');
+        const alertMessage = generateAlertMessage(alertType, piUsername, {
+            ...metadata,
+            isTest: true,
+            testSentBy: Number(userCid),
+            testSentAt: new Date().toISOString()
+        });
+        
+        // Create notifications for each target user (using Firebase UIDs)
+        const batch = db.batch();
+        const notificationRefs = [];
+        const counterUpdates = {};
+        const today = new Date().toISOString().split('T')[0];
+        
+        for (const firebaseUid of targetFirebaseUids) {
+            const notificationId = `test_alert_${Date.now()}_${firebaseUid}_${piCid}_${Math.random().toString(36).substring(2, 9)}`;
+            const notificationRef = db.collection('user_notifications')
+                .doc(firebaseUid) // Use Firebase UID, not eToro CID
+                .collection('notifications')
+                .doc(notificationId);
+            
+            const notificationData = {
+                id: notificationId,
+                type: 'alert',
+                title: `[TEST] ${alertType.name}`,
+                message: alertMessage,
+                read: false,
+                createdAt: FieldValue.serverTimestamp(),
+                timestamp: FieldValue.serverTimestamp(), // Also include timestamp for ordering
+                metadata: {
+                    piCid: Number(piCid),
+                    piUsername: piUsername,
+                    alertType: alertType.id,
+                    alertTypeName: alertType.name,
+                    computationName: alertType.computationName,
+                    computationDate: today,
+                    severity: alertType.severity,
+                    isTest: true,
+                    testSentBy: Number(userCid),
+                    testSentAt: new Date().toISOString(),
+                    ...metadata
+                }
+            };
+            
+            batch.set(notificationRef, notificationData);
+            notificationRefs.push(notificationRef);
+            
+            // Track counter updates (using Firebase UID as key)
+            if (!counterUpdates[firebaseUid]) {
+                counterUpdates[firebaseUid] = {
+                    date: today,
+                    unreadCount: 0,
+                    totalCount: 0,
+                    byType: {}
+                };
+            }
+            counterUpdates[firebaseUid].unreadCount += 1;
+            counterUpdates[firebaseUid].totalCount += 1;
+            counterUpdates[firebaseUid].byType[alertType.id] = 
+                (counterUpdates[firebaseUid].byType[alertType.id] || 0) + 1;
+        }
+        
+        // Update notification counters (using Firebase UIDs)
+        for (const [firebaseUid, counter] of Object.entries(counterUpdates)) {
+            const counterRef = db.collection('user_notifications')
+                .doc(firebaseUid) // Use Firebase UID, not eToro CID
+                .collection('counters')
+                .doc(counter.date);
+            
+            batch.set(counterRef, {
+                date: counter.date,
+                unreadCount: FieldValue.increment(counter.unreadCount),
+                totalCount: FieldValue.increment(counter.totalCount),
+                [`byType.${alertType.id}`]: FieldValue.increment(counter.byType[alertType.id] || 0),
+                lastUpdated: FieldValue.serverTimestamp()
+            }, { merge: true });
+        }
+        
+        // Commit batch
+        await batch.commit();
+        
+        logger.log('SUCCESS', `[sendTestAlert] Created ${notificationRefs.length} test notifications for alert type ${alertType.id}`);
+        
+        return res.status(200).json({
+            success: true,
+            message: `Test alert sent to ${targetFirebaseUids.length} users`,
+            alertType: {
+                id: alertType.id,
+                name: alertType.name,
+                computationName: alertType.computationName
+            },
+            targetUsers: {
+                count: targetFirebaseUids.length,
+                firebaseUids: targetFirebaseUids
+            },
+            piCid: Number(piCid),
+            piUsername: piUsername,
+            notificationsCreated: notificationRefs.length
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[sendTestAlert] Error sending test alert:`, error);
+        return res.status(500).json({ error: error.message });
+    }
+}
+
+module.exports = { sendTestAlert };
+

package/functions/generic-api/user-api/index.js

@@ -12,6 +12,7 @@ const { setDevOverride, getDevOverrideStatus } = require('./helpers/dev_helpers'
 const { getAlertTypes, getDynamicWatchlistComputations, getUserAlerts, getAlertCount, markAlertRead, markAllAlertsRead, deleteAlert } = require('./helpers/alert_helpers');
 const { requestPiFetch, getPiFetchStatus } = require('./helpers/on_demand_fetch_helpers');
 const { requestUserSync, getUserSyncStatus } = require('./helpers/user_sync_helpers');
+const { sendTestAlert } = require('./helpers/test_alert_helpers');
 
 module.exports = (dependencies, config) => {
     const router = express.Router();
@@ -87,6 +88,7 @@ module.exports = (dependencies, config) => {
     // --- Developer Mode (only for whitelisted developer accounts) ---
     router.post('/dev/override', (req, res) => setDevOverride(req, res, dependencies, config));
     router.get('/dev/override', (req, res) => getDevOverrideStatus(req, res, dependencies, config));
+    router.post('/dev/test-alert', (req, res) => sendTestAlert(req, res, dependencies, config));
     
     // --- Alert Management ---
     router.get('/me/alert-types', (req, res) => getAlertTypes(req, res, dependencies, config));

package/functions/root-data-indexer/index.js

@@ -270,19 +270,33 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             ]);
             
             // Check if date exists in tracking documents
+            // The _dates document uses dot notation: fetchedDates.2025-12-29: true
+            // When read, this becomes: { fetchedDates: { "2025-12-29": true } }
             let foundPISocial = false;
             let foundSignedInSocial = false;
             
             if (piSocialTrackingDoc.exists) {
                 const data = piSocialTrackingDoc.data();
-                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                // Check both nested structure and flat dot-notation structure
+                if (data.fetchedDates && typeof data.fetchedDates === 'object') {
+                    if (data.fetchedDates[dateStr] === true) {
+                        foundPISocial = true;
+                    }
+                } else if (data[`fetchedDates.${dateStr}`] === true) {
+                    // Handle flat dot-notation structure (if Firestore stores it that way)
                     foundPISocial = true;
                 }
             }
             
             if (signedInSocialTrackingDoc.exists) {
                 const data = signedInSocialTrackingDoc.data();
-                if (data.fetchedDates && data.fetchedDates[dateStr] === true) {
+                // Check both nested structure and flat dot-notation structure
+                if (data.fetchedDates && typeof data.fetchedDates === 'object') {
+                    if (data.fetchedDates[dateStr] === true) {
+                        foundSignedInSocial = true;
+                    }
+                } else if (data[`fetchedDates.${dateStr}`] === true) {
+                    // Handle flat dot-notation structure (if Firestore stores it that way)
                     foundSignedInSocial = true;
                 }
             } 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.493",
+  "version": "1.0.495",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [