bulltrackers-module 1.0.464 → 1.0.466

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,5 +1,5 @@
-# Data Feeder Pipeline (Final Verified Version)
-# Orchestrates data fetching with UTC-alignment, Test Mode, and extended Timeouts.
+# Data Feeder Pipeline (V2.3 - Try/Retry Syntax Fixed)
+# Orchestrates data fetching with UTC-alignment, Test Mode, and Reliability.
 
 main:
   params: [input]
@@ -9,9 +9,16 @@ main:
           - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
           - location: "europe-west1"
           - market_date: '${text.split(time.format(sys.now()), "T")[0]}'
+          # Define a central retry policy to reuse across all HTTP calls
+          - default_retry:
+              predicate: ${http.default_retry_predicate}
+              max_retries: 5
+              backoff:
+                initial_delay: 2
+                max_delay: 60
+                multiplier: 2
 
     # --- TEST MODE / SELECTIVE EXECUTION ---
-    # Static routing to allow manual testing of specific steps via Input JSON.
     - check_test_mode:
         switch:
           - condition: '${input != null and "target_step" in input}'
@@ -34,33 +41,36 @@ main:
             - price_fetch:
                 steps:
                   - call_price_fetcher:
-                      call: http.post
-                      args:
-                        url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
-                        auth: { type: OIDC }
-                        timeout: 300 # Fixed: Prevents timeout errors for long-running functions
-                      retry: ${http.default_retry}
+                      try:
+                        call: http.post
+                        args:
+                          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
+                          auth: { type: OIDC }
+                          timeout: 300
+                      retry: ${default_retry} # Fixed: Moved retry to a Try Step
             - insights_fetch:
                 steps:
                   - call_insights_fetcher:
-                      call: http.post
-                      args:
-                        url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
-                        auth: { type: OIDC }
-                        timeout: 300 # Fixed: Prevents timeout errors for long-running functions
-                      retry: ${http.default_retry}
+                      try:
+                        call: http.post
+                        args:
+                          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
+                          auth: { type: OIDC }
+                          timeout: 300
+                      retry: ${default_retry} # Fixed: Moved retry to a Try Step
 
     - index_market_data:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body:
-            targetDate: '${market_date}'
-          auth: { type: OIDC }
-          timeout: 300
+        try:
+          call: http.post
+          args:
+            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+            body:
+              targetDate: '${market_date}'
+            auth: { type: OIDC }
+            timeout: 300
+        retry: ${default_retry}
 
     # --- PHASE 2: ALIGN TO MIDNIGHT ---
-    # Dynamically calculates seconds remaining until exactly 00:00 UTC.
     - wait_for_midnight:
         assign:
           - now_sec: '${int(sys.now())}'
@@ -79,6 +89,7 @@ main:
             url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
             auth: { type: OIDC }
             timeout: 300
+        retry: ${default_retry}
         except:
           as: e
           steps:
@@ -89,31 +100,36 @@ main:
                   text: '${"Rankings Fetch Failed: " + json.encode(e)}'
 
     - run_social_midnight:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
-          auth: { type: OIDC }
-          timeout: 300
+        try:
+          call: http.post
+          args:
+            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
+            auth: { type: OIDC }
+            timeout: 300
+        retry: ${default_retry}
 
-    # Fixed: Split assign and call into two separate steps
     - prepare_midnight_index:
         assign:
             - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
     - index_midnight_data:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          body:
-             targetDate: '${current_date}'
-          auth: { type: OIDC }
-          timeout: 300
+        try:
+          call: http.post
+          args:
+            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+            body:
+               targetDate: '${current_date}'
+            auth: { type: OIDC }
+            timeout: 300
+        retry: ${default_retry}
 
     - run_global_indexer:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          auth: { type: OIDC }
-          timeout: 300
+        try:
+          call: http.post
+          args:
+            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+            auth: { type: OIDC }
+            timeout: 300
+        retry: ${default_retry}
 
     # --- PHASE 4: RECURRING SOCIAL FETCH (UTC Aligned 3hr) ---
     - init_social_loop:
@@ -127,7 +143,7 @@ main:
               - calculate_next_window:
                   assign:
                     - now_sec_loop: '${int(sys.now())}'
-                    - window_size: 10800 # Fixed: Aligns to 03:00, 06:00, 09:00 UTC...
+                    - window_size: 10800 
                     - sleep_loop: '${window_size - (now_sec_loop % window_size)}'
               
               - wait_for_3hr_window:
@@ -136,24 +152,27 @@ main:
                     seconds: '${sleep_loop}'
               
               - run_social_recurring:
-                  call: http.post
-                  args:
-                    url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
-                    auth: { type: OIDC }
-                    timeout: 300
+                  try:
+                    call: http.post
+                    args:
+                      url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
+                      auth: { type: OIDC }
+                      timeout: 300
+                  retry: ${default_retry}
 
-              # Fixed: Split assign and call into two separate steps
               - prepare_recurring_index:
                    assign:
                      - cur_date_rec: '${text.split(time.format(sys.now()), "T")[0]}'
               - index_recurring:
-                   call: http.post
-                   args:
-                     url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-                     body:
-                       targetDate: '${cur_date_rec}'
-                     auth: { type: OIDC }
-                     timeout: 300
+                   try:
+                     call: http.post
+                     args:
+                       url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+                       body:
+                         targetDate: '${cur_date_rec}'
+                       auth: { type: OIDC }
+                       timeout: 300
+                   retry: ${default_retry}
               
               - increment:
                   assign:

package/functions/generic-api/user-api/helpers/alert_helpers.js

@@ -20,7 +20,8 @@ async function getAlertTypes(req, res, dependencies, config) {
         id: type.id,
         name: type.name,
         description: type.description,
-        severity: type.severity
+        severity: type.severity,
+        computationName: type.computationName // Include computation name for dynamic watchlists
       }))
     });
   } catch (error) {
@@ -30,6 +31,33 @@ async function getAlertTypes(req, res, dependencies, config) {
   }
 }
 
+/**
+ * GET /user/me/dynamic-watchlist-computations
+ * Get available computations for dynamic watchlists (from alert types)
+ */
+async function getDynamicWatchlistComputations(req, res, dependencies, config) {
+  try {
+    const alertTypes = getAllAlertTypes();
+    
+    // Extract unique computations from alert types
+    const computations = alertTypes.map(type => ({
+      computationName: type.computationName,
+      alertTypeName: type.name,
+      description: type.description,
+      severity: type.severity
+    }));
+    
+    return res.status(200).json({
+      success: true,
+      computations
+    });
+  } catch (error) {
+    const { logger } = dependencies;
+    logger.log('ERROR', '[getDynamicWatchlistComputations] Error fetching computations', error);
+    return res.status(500).json({ error: error.message });
+  }
+}
+
 /**
  * GET /user/me/alerts
  * Get user's alerts (paginated)
@@ -317,6 +345,7 @@ async function deleteAlert(req, res, dependencies, config) {
 
 module.exports = {
   getAlertTypes,
+  getDynamicWatchlistComputations,
   getUserAlerts,
   getAlertCount,
   markAlertRead,

package/functions/generic-api/user-api/helpers/watchlist_helpers.js

@@ -148,20 +148,7 @@ async function createWatchlist(req, res, dependencies, config) {
         
         await watchlistRef.set(watchlistData);
         
-        // If public, also add to public watchlists collection
-        if (visibility === 'public') {
-            const publicRef = db.collection('public_watchlists').doc(watchlistId);
-            await publicRef.set({
-                watchlistId,
-                createdBy: Number(userCid),
-                name: watchlistData.name,
-                type,
-                description: dynamicConfig?.description || '',
-                copyCount: 0,
-                createdAt: FieldValue.serverTimestamp(),
-                updatedAt: FieldValue.serverTimestamp()
-            });
-        }
+        // Watchlists are always created as private - no public version on creation
         
         logger.log('SUCCESS', `[createWatchlist] Created ${type} watchlist "${name}" for user ${userCid}`);
         
@@ -250,29 +237,12 @@ async function updateWatchlist(req, res, dependencies, config) {
             updates.hasBeenModified = true;
         }
         
-        if (visibility !== undefined) {
-            if (visibility !== 'public' && visibility !== 'private') {
-                return res.status(400).json({ error: "Visibility must be 'public' or 'private'" });
-            }
-            updates.visibility = visibility;
-            
-            // Update public watchlists collection
-            const publicRef = db.collection('public_watchlists').doc(id);
-            if (visibility === 'public') {
-                await publicRef.set({
-                    watchlistId: id,
-                    createdBy: existingData.createdBy,
-                    name: updates.name || existingData.name,
-                    type: existingData.type,
-                    description: (dynamicConfig?.description || existingData.dynamicConfig?.description || ''),
-                    copyCount: existingData.copyCount || 0,
-                    createdAt: existingData.createdAt,
-                    updatedAt: FieldValue.serverTimestamp()
-                }, { merge: true });
-            } else {
-                // Remove from public if making private
-                await publicRef.delete();
-            }
+        // Visibility changes are now handled through publish/unpublish endpoints
+        // Users can only update their private watchlist
+        if (visibility !== undefined && visibility !== 'private') {
+            return res.status(400).json({ 
+                error: "Cannot change visibility directly. Use the publish endpoint to create a public version." 
+            });
         }
         
         if (items !== undefined && existingData.type === 'static') {
@@ -368,7 +338,7 @@ async function copyWatchlist(req, res, dependencies, config) {
     const { db, logger } = dependencies;
     const { userCid } = req.query;
     const { id } = req.params;
-    const { name } = req.body; // Optional custom name
+    const { name, version } = req.body; // Optional custom name and version number
     
     if (!userCid || !id) {
         return res.status(400).json({ error: "Missing userCid or watchlist id" });
@@ -388,20 +358,51 @@ async function copyWatchlist(req, res, dependencies, config) {
         const publicData = publicDoc.data();
         const originalCreatorCid = Number(publicData.createdBy);
         
-        // Find the original watchlist
-        const originalRef = db.collection(config.watchlistsCollection || 'watchlists')
-            .doc(String(originalCreatorCid))
-            .collection('lists')
-            .doc(id);
-        
-        const originalDoc = await originalRef.get();
+        let originalData;
         
-        if (!originalDoc.exists) {
-            return res.status(404).json({ error: "Original watchlist not found" });
+        // If version is specified, copy from that specific version snapshot
+        if (version) {
+            const versionRef = db.collection('public_watchlists')
+                .doc(id)
+                .collection('versions')
+                .doc(String(version));
+            
+            const versionDoc = await versionRef.get();
+            
+            if (!versionDoc.exists) {
+                return res.status(404).json({ error: `Version ${version} not found for this watchlist` });
+            }
+            
+            originalData = versionDoc.data();
+        } else {
+            // Copy from latest version (get the latest version)
+            const latestVersion = publicData.latestVersion || 1;
+            const versionRef = db.collection('public_watchlists')
+                .doc(id)
+                .collection('versions')
+                .doc(String(latestVersion));
+            
+            const versionDoc = await versionRef.get();
+            
+            if (versionDoc.exists) {
+                originalData = versionDoc.data();
+            } else {
+                // Fallback: try to get from original watchlist (for backwards compatibility)
+                const originalRef = db.collection(config.watchlistsCollection || 'watchlists')
+                    .doc(String(originalCreatorCid))
+                    .collection('lists')
+                    .doc(id);
+                
+                const originalDoc = await originalRef.get();
+                
+                if (!originalDoc.exists) {
+                    return res.status(404).json({ error: "Original watchlist not found" });
+                }
+                
+                originalData = originalDoc.data();
+            }
         }
         
-        const originalData = originalDoc.data();
-        
         // Check if user is copying their own watchlist
         const isCopyingOwn = originalCreatorCid === userCidNum;
         
@@ -445,6 +446,7 @@ async function copyWatchlist(req, res, dependencies, config) {
             createdBy: userCidNum,
             visibility: 'private', // Copied watchlists are always private
             copiedFrom: id,
+            copiedFromVersion: version || (publicData.latestVersion || 1), // Track which version was copied
             copiedFromCreator: originalCreatorCid,
             originalName: originalData.name, // Store original name for comparison
             hasBeenModified: false, // Track if user made meaningful changes
@@ -455,6 +457,11 @@ async function copyWatchlist(req, res, dependencies, config) {
         
         // Remove fields that shouldn't be copied
         delete watchlistData.copyCount;
+        delete watchlistData.version;
+        delete watchlistData.versionId;
+        delete watchlistData.snapshotAt;
+        delete watchlistData.isImmutable;
+        delete watchlistData.watchlistId;
         
         const newWatchlistRef = db.collection(config.watchlistsCollection || 'watchlists')
             .doc(String(userCidNum))
@@ -525,6 +532,150 @@ async function getPublicWatchlists(req, res, dependencies, config) {
     }
 }
 
+/**
+ * POST /user/me/watchlists/:id/publish
+ * Publish a version of a private watchlist
+ * Creates an immutable snapshot version that can be copied by others
+ */
+async function publishWatchlistVersion(req, res, dependencies, config) {
+    const { db, logger } = dependencies;
+    const { userCid } = req.query;
+    const { id } = req.params;
+    
+    if (!userCid || !id) {
+        return res.status(400).json({ error: "Missing userCid or watchlist id" });
+    }
+    
+    try {
+        const watchlistsCollection = config.watchlistsCollection || 'watchlists';
+        const watchlistRef = db.collection(watchlistsCollection)
+            .doc(String(userCid))
+            .collection('lists')
+            .doc(id);
+        
+        const watchlistDoc = await watchlistRef.get();
+        
+        if (!watchlistDoc.exists) {
+            return res.status(404).json({ error: "Watchlist not found" });
+        }
+        
+        const watchlistData = watchlistDoc.data();
+        
+        // Verify ownership
+        if (watchlistData.createdBy !== Number(userCid)) {
+            return res.status(403).json({ error: "You can only publish your own watchlists" });
+        }
+        
+        // Check if this is a copied watchlist that hasn't been modified
+        if (watchlistData.copiedFrom && watchlistData.copiedFromCreator && !watchlistData.hasBeenModified) {
+            return res.status(400).json({ 
+                error: "Cannot publish copied watchlist without making meaningful changes. Please modify the watchlist items, thresholds, or parameters before publishing." 
+            });
+        }
+        
+        // Get current version number
+        const publicRef = db.collection('public_watchlists').doc(id);
+        const publicDoc = await publicRef.get();
+        
+        let versionNumber = 1;
+        if (publicDoc.exists) {
+            const publicData = publicDoc.data();
+            versionNumber = (publicData.latestVersion || 0) + 1;
+        }
+        
+        // Create version snapshot
+        const versionId = `${id}_v${versionNumber}`;
+        const versionData = {
+            watchlistId: id,
+            version: versionNumber,
+            createdBy: Number(userCid),
+            name: watchlistData.name,
+            type: watchlistData.type,
+            description: watchlistData.dynamicConfig?.description || '',
+            // Snapshot the current state
+            items: watchlistData.items ? JSON.parse(JSON.stringify(watchlistData.items)) : undefined,
+            dynamicConfig: watchlistData.dynamicConfig ? JSON.parse(JSON.stringify(watchlistData.dynamicConfig)) : undefined,
+            snapshotAt: FieldValue.serverTimestamp(),
+            copyCount: 0,
+            createdAt: watchlistData.createdAt,
+            isImmutable: true // Public versions are immutable
+        };
+        
+        // Store version in versions subcollection
+        const versionRef = db.collection('public_watchlists')
+            .doc(id)
+            .collection('versions')
+            .doc(String(versionNumber));
+        
+        await versionRef.set(versionData);
+        
+        // Update or create public watchlist entry (points to latest version)
+        await publicRef.set({
+            watchlistId: id,
+            createdBy: Number(userCid),
+            name: watchlistData.name,
+            type: watchlistData.type,
+            description: watchlistData.dynamicConfig?.description || '',
+            latestVersion: versionNumber,
+            latestVersionId: versionId,
+            copyCount: publicDoc.exists ? (publicDoc.data().copyCount || 0) : 0,
+            createdAt: watchlistData.createdAt,
+            updatedAt: FieldValue.serverTimestamp()
+        }, { merge: true });
+        
+        logger.log('SUCCESS', `[publishWatchlistVersion] User ${userCid} published watchlist ${id} as version ${versionNumber}`);
+        
+        return res.status(201).json({
+            success: true,
+            version: versionNumber,
+            versionId: versionId,
+            watchlist: versionData
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[publishWatchlistVersion] Error publishing watchlist ${id} for ${userCid}`, error);
+        return res.status(500).json({ error: error.message });
+    }
+}
+
+/**
+ * GET /user/public-watchlists/:id/versions
+ * Get version history for a public watchlist
+ */
+async function getWatchlistVersions(req, res, dependencies, config) {
+    const { db, logger } = dependencies;
+    const { id } = req.params;
+    
+    try {
+        const versionsRef = db.collection('public_watchlists')
+            .doc(id)
+            .collection('versions')
+            .orderBy('version', 'desc');
+        
+        const snapshot = await versionsRef.get();
+        const versions = [];
+        
+        snapshot.forEach(doc => {
+            versions.push({
+                version: doc.data().version,
+                versionId: `${id}_v${doc.data().version}`,
+                ...doc.data()
+            });
+        });
+        
+        return res.status(200).json({
+            success: true,
+            watchlistId: id,
+            versions,
+            count: versions.length
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[getWatchlistVersions] Error fetching versions for ${id}`, error);
+        return res.status(500).json({ error: error.message });
+    }
+}
+
 module.exports = {
     getUserWatchlists,
     getWatchlist,
@@ -532,6 +683,8 @@ module.exports = {
     updateWatchlist,
     deleteWatchlist,
     copyWatchlist,
-    getPublicWatchlists
+    getPublicWatchlists,
+    publishWatchlistVersion,
+    getWatchlistVersions
 };
 

package/functions/generic-api/user-api/index.js

@@ -6,10 +6,10 @@ const express = require('express');
 const { submitReview, getReviews, getUserReview, checkReviewEligibility } = require('./helpers/review_helpers');
 const { getPiAnalytics, getUserRecommendations, getWatchlist, updateWatchlist, autoGenerateWatchlist, getUserDataStatus, getUserPortfolio, getUserSocialPosts, getUserComputations, getUserVerification, getInstrumentMappings, searchPopularInvestors, requestPiAddition, getWatchlistTriggerCounts, checkPisInRankings, getPiProfile, checkIfUserIsPopularInvestor, trackProfileView, getSignedInUserPIPersonalizedMetrics } = require('./helpers/data_helpers');
 const { initiateVerification, finalizeVerification } = require('./helpers/verification_helpers');
-const { getUserWatchlists, getWatchlist: getWatchlistById, createWatchlist, updateWatchlist: updateWatchlistById, deleteWatchlist, copyWatchlist, getPublicWatchlists } = require('./helpers/watchlist_helpers');
+const { getUserWatchlists, getWatchlist: getWatchlistById, createWatchlist, updateWatchlist: updateWatchlistById, deleteWatchlist, copyWatchlist, getPublicWatchlists, publishWatchlistVersion, getWatchlistVersions } = require('./helpers/watchlist_helpers');
 const { subscribeToAlerts, updateSubscription, unsubscribeFromAlerts, getUserSubscriptions, subscribeToWatchlist } = require('./helpers/subscription_helpers');
 const { setDevOverride, getDevOverrideStatus } = require('./helpers/dev_helpers');
-const { getAlertTypes, getUserAlerts, getAlertCount, markAlertRead, markAllAlertsRead, deleteAlert } = require('./helpers/alert_helpers');
+const { getAlertTypes, getDynamicWatchlistComputations, getUserAlerts, getAlertCount, markAlertRead, markAllAlertsRead, deleteAlert } = require('./helpers/alert_helpers');
 const { requestPiFetch, getPiFetchStatus } = require('./helpers/on_demand_fetch_helpers');
 const { requestUserSync, getUserSyncStatus } = require('./helpers/user_sync_helpers');
 
@@ -65,9 +65,11 @@ module.exports = (dependencies, config) => {
     router.put('/me/watchlists/:id', (req, res) => updateWatchlistById(req, res, dependencies, config));
     router.delete('/me/watchlists/:id', (req, res) => deleteWatchlist(req, res, dependencies, config));
     router.post('/me/watchlists/:id/copy', (req, res) => copyWatchlist(req, res, dependencies, config));
+    router.post('/me/watchlists/:id/publish', (req, res) => publishWatchlistVersion(req, res, dependencies, config));
     
     // Public watchlists
     router.get('/public-watchlists', (req, res) => getPublicWatchlists(req, res, dependencies, config));
+    router.get('/public-watchlists/:id/versions', (req, res) => getWatchlistVersions(req, res, dependencies, config));
     
     // --- Alert Subscriptions ---
     router.post('/me/subscriptions', (req, res) => subscribeToAlerts(req, res, dependencies, config));
@@ -88,6 +90,7 @@ module.exports = (dependencies, config) => {
     
     // --- Alert Management ---
     router.get('/me/alert-types', (req, res) => getAlertTypes(req, res, dependencies, config));
+    router.get('/me/dynamic-watchlist-computations', (req, res) => getDynamicWatchlistComputations(req, res, dependencies, config));
     router.get('/me/alerts', (req, res) => getUserAlerts(req, res, dependencies, config));
     router.get('/me/alerts/count', (req, res) => getAlertCount(req, res, dependencies, config));
     router.put('/me/alerts/:alertId/read', (req, res) => markAlertRead(req, res, dependencies, config));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.464",
+  "version": "1.0.466",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [