bulltrackers-module 1.0.404 → 1.0.406

package/functions/computation-system/context/ContextFactory.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Factory for creating the Computation Context.
  * UPDATED: Injects verification and rankings data into context globally and locally.
+ * UPDATED: Added support for historical ranking data in both Standard and Meta contexts.
  */
 const mathLayer          = require('../layers/index');
 const { LEGACY_MAPPING } = require('../topology/HashManager');
@@ -21,8 +22,10 @@ class ContextFactory {
             todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, 
             userId, userType, dateStr, metadata, mappings, insights, socialData, 
             computedDependencies, previousComputedDependencies, config, deps,
-            verification, rankings, // User-specific
-            allRankings, allVerifications // [NEW] Global context for comparative analysis
+            verification, 
+            rankings, yesterdayRankings,         // User-specific rank entries
+            allRankings, allRankingsYesterday,   // Global rank lists
+            allVerifications 
         } = options;
         
         return {
@@ -32,7 +35,8 @@ class ContextFactory {
                 portfolio: { today: todayPortfolio, yesterday: yesterdayPortfolio }, 
                 history: { today: todayHistory, yesterday: yesterdayHistory },
                 verification: verification || null,
-                rankEntry: rankings || null // Renamed for clarity (single entry)
+                rankEntry: rankings || null,
+                rankEntryYesterday: yesterdayRankings || null
             },
             date: { today: dateStr },
             insights: { today: insights?.today, yesterday: insights?.yesterday },
@@ -42,9 +46,9 @@ class ContextFactory {
             computed: computedDependencies || {},
             previousComputed: previousComputedDependencies || {},
             meta: metadata, config, deps,
-            // [NEW] Global Data Injection for Comparative Logic (Similarity, Ranking position, etc.)
             globalData: {
                 rankings: allRankings || [],
+                rankingsYesterday: allRankingsYesterday || [],
                 verifications: allVerifications || {}
             }
         };
@@ -54,7 +58,8 @@ class ContextFactory {
         const { 
             dateStr, metadata, mappings, insights, socialData, prices, 
             computedDependencies, previousComputedDependencies, config, deps,
-            allRankings, allVerifications 
+            allRankings, allRankingsYesterday, // [UPDATED] Accepted here
+            allVerifications 
         } = options;
         
         return {
@@ -69,6 +74,7 @@ class ContextFactory {
             meta: metadata, config, deps,
             globalData: {
                 rankings: allRankings || [],
+                rankingsYesterday: allRankingsYesterday || [], // [UPDATED] Injected here
                 verifications: allVerifications || {}
             }
         };

package/functions/computation-system/executors/MetaExecutor.js

@@ -3,6 +3,7 @@
  * UPDATED: Uses CachedDataLoader for all data access.
  * UPDATED: Tracks processed shard/item counts.
  * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
+ * UPDATED: Support for historical rankings in Meta Context.
  */
 const { normalizeName }    = require('../utils/utils');
 const { CachedDataLoader } = require('../data/CachedDataLoader');
@@ -13,9 +14,20 @@ class MetaExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
         const dStr = date.toISOString().slice(0, 10);
         const { logger, db } = deps;
-        const { CachedDataLoader } = require('../data/CachedDataLoader');
         const loader = new CachedDataLoader(config, deps);
 
+        // [FIX] Check if any meta calculation needs history
+        const needsHistory = calcs.some(c => c.isHistorical);
+        let rankingsYesterday = null;
+
+        if (needsHistory) {
+             const prevDate = new Date(date);
+             prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+             const prevStr = prevDate.toISOString().slice(0, 10);
+             logger.log('INFO', `[MetaExecutor] Loading historical rankings for ${prevStr}`);
+             rankingsYesterday = await loader.loadRankings(prevStr);
+        }
+
         // 1. Load Global Dependencies
         const [mappings, rankings, verifications] = await Promise.all([
             loader.loadMappings(),
@@ -38,11 +50,14 @@ class MetaExecutor {
                 previousComputedDependencies: previousFetchedDeps,
                 config, deps,
                 allRankings: rankings,
+                allRankingsYesterday: rankingsYesterday, // [FIX] Injected
                 allVerifications: verifications
             });
 
             try {
                 const result = await inst.process(context);
+                // Meta results are usually wrapped in a global key or just the result object
+                // The structure below implies we store it under the date key
                 inst.results = { [dStr]: { global: result } };
                 state[c.name] = inst;
             } catch (e) {
@@ -61,6 +76,18 @@ class MetaExecutor {
         const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
         
+        // [FIX] Historical support for Batch/OncePerDay execution
+        let rankingsYesterday = null;
+        if (metadata.isHistorical) {
+             const prevDate = new Date(dateStr);
+             prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+             const prevStr = prevDate.toISOString().slice(0, 10);
+             rankingsYesterday = await loader.loadRankings(prevStr);
+        }
+
+        // Load current rankings (often needed for ContextFactory.buildMetaContext)
+        const rankings = await loader.loadRankings(dateStr);
+
         if (metadata.rootDataDependencies?.includes('price')) {
             logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
             const shardRefs = await loader.getPriceShardReferences();
@@ -72,7 +99,9 @@ class MetaExecutor {
                 const partialContext = ContextFactory.buildMetaContext({ 
                     dateStr, metadata, mappings, insights, socialData: social, 
                     prices: { history: shardData }, computedDependencies: computedDeps, 
-                    previousComputedDependencies: prevDeps, config, deps 
+                    previousComputedDependencies: prevDeps, config, deps,
+                    allRankings: rankings,
+                    allRankingsYesterday: rankingsYesterday
                 });
                 
                 await calcInstance.process(partialContext);
@@ -90,7 +119,9 @@ class MetaExecutor {
             const context = ContextFactory.buildMetaContext({ 
                 dateStr, metadata, mappings, insights, socialData: social, 
                 prices: {}, computedDependencies: computedDeps, 
-                previousComputedDependencies: prevDeps, config, deps 
+                previousComputedDependencies: prevDeps, config, deps,
+                allRankings: rankings,
+                allRankingsYesterday: rankingsYesterday
             });
             const res = await calcInstance.process(context);
             
@@ -102,4 +133,4 @@ class MetaExecutor {
     }
 }
 
-module.exports = { MetaExecutor };
+module.exports = { MetaExecutor };

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,7 +1,7 @@
 /**
  * {
  * type: uploaded file
- * fileName: bulltrackers-module/functions/computation-system/executors/StandardExecutor.js
+ * fileName: computation-system/executors/StandardExecutor.js
  * }
  */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
@@ -141,6 +141,7 @@ class StandardExecutor {
     }
 
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
+        // ... (No changes to flushBuffer)
         const transformedState = {};
         for (const [name, inst] of Object.entries(state)) {
             const rawResult = inst.results || {};
@@ -178,6 +179,7 @@ class StandardExecutor {
     }
 
     static mergeReports(successAcc, failureAcc, newResult) {
+        // ... (No changes to mergeReports)
         if (!newResult) return;
         for (const [name, update] of Object.entries(newResult.successUpdates)) {
             if (!successAcc[name]) {
@@ -218,8 +220,15 @@ class StandardExecutor {
         const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
         const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
         
-        // [UPDATED] Load the Partitioned Social Data Container
-        // socialContainer = { generic: {}, pi: { userId: {} }, signedIn: { userId: {} } }
+        // [FIX] Load Yesterday's Rankings if isHistorical is true
+        let yesterdayRankings = null;
+        if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
+             const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+             const prevStr = prevDate.toISOString().slice(0, 10);
+             // Assuming CachedDataLoader handles caching for efficiency
+             yesterdayRankings = await loader.loadRankings(prevStr); 
+        }
+
         const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
 
         let chunkSuccess = 0;
@@ -229,7 +238,6 @@ class StandardExecutor {
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
             const todayHistory = historyData ? historyData[userId] : null;
             
-            // 2. Identify User Type
             let actualUserType = todayPortfolio._userType;
             if (!actualUserType) {
                 if (todayPortfolio.PublicPositions) {
@@ -240,7 +248,6 @@ class StandardExecutor {
                 }
             }
             
-            // 3. Strict User Type Filtering
             if (targetUserType && targetUserType !== 'all') {
                  if (targetUserType !== actualUserType) {
                      if (stats) stats.skippedUsers++;
@@ -248,22 +255,19 @@ class StandardExecutor {
                  }
             }
             
-            // 4. Resolve Contextual Data
             const userVerification = verifications ? verifications[userId] : null;
+            
+            // [FIX] Extract current AND yesterday's rank entry for this user
             const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
+            const userRankingYesterday = yesterdayRankings ? (yesterdayRankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
 
-            // [CRITICAL FIX] Select Specific Social Data to Prevent Skew
             let effectiveSocialData = null;
             if (socialContainer) {
                 if (actualUserType === 'POPULAR_INVESTOR') {
-                    // PI Route: Only gets PI-specific posts for this user
-                    // (PIs do not need generic noise usually, or you can merge it if required)
                     effectiveSocialData = socialContainer.pi[userId] || {}; 
                 } else if (actualUserType === 'SIGNED_IN_USER') {
-                    // Signed-In Route: Only gets Signed-In specific posts for this user
                     effectiveSocialData = socialContainer.signedIn[userId] || {};
                 } else {
-                    // Normal/Speculator/All Route: Gets Generic Market Posts
                     effectiveSocialData = socialContainer.generic || {};
                 }
             }
@@ -271,15 +275,19 @@ class StandardExecutor {
             const context = ContextFactory.buildPerUserContext({ 
                 todayPortfolio, yesterdayPortfolio, todayHistory, userId, 
                 userType: actualUserType, dateStr, metadata, mappings, insights, 
-                
-                // Inject the filtered subset
                 socialData: effectiveSocialData ? { today: effectiveSocialData } : null,
-                
                 computedDependencies: computedDeps, previousComputedDependencies: prevDeps, 
                 config, deps,
                 verification: userVerification, 
+                
+                // [FIX] Pass both ranking entries
                 rankings: userRanking,
+                yesterdayRankings: userRankingYesterday, 
+                
+                // [FIX] Pass both global lists
                 allRankings: rankings,
+                allRankingsYesterday: yesterdayRankings,
+                
                 allVerifications: verifications
             });
 

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -202,90 +202,66 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
             logger.log('INFO', `[PI Update] Deep portfolio fetching is disabled. Skipping deep positions for ${username}`);
         }
 
-        // --- 3. Fetch Trade History (Last 1 Year) with Pagination ---
+        // --- 3. Fetch Trade History (Last 1 Year) - Single Page with Large ItemsPerPage ---
         const oneYearAgo = new Date();
         oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
         
         const historyBaseUrl = ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
+        const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
         
-        let allHistoryTrades = [];
-        let page = 1;
-        let hasMore = true;
-        const MAX_PAGES = 10; 
-
-        while (hasMore && page <= MAX_PAGES) {
-             const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=${page}&ItemsPerPage=1000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
-             
-             logger.log('INFO', `[PI DEBUG] Fetching History Page ${page} for ${username} URL: ${historyUrl}`);
-
-             try {
-                // Reuse headers
-                let historyRes = null;
-                let historySuccess = false;
-                
-                // [CIRCUIT BREAKER] Try proxy only if circuit is not open
-                if (shouldTryProxy()) {
-                    try {
-                        historyRes = await proxyManager.fetch(historyUrl, requestOptions);
-                        if (historyRes.ok) {
-                            historySuccess = true;
-                            recordProxyOutcome(true);
-                        } else {
-                            throw new Error(`Proxy returned non-ok status: ${historyRes.status}`);
-                        }
-                    } catch (proxyError) {
-                        recordProxyOutcome(false);
-                        logger.log('WARN', `[PI Update] Proxy failed for history page ${page} (${getFailureCount()}/${getMaxFailures()}). Trying direct...`);
-                    }
+        logger.log('INFO', `[PI DEBUG] Fetching History for ${username} URL: ${historyUrl}`);
+
+        let historyRes = null;
+        let historySuccess = false;
+        
+        // [CIRCUIT BREAKER] Try proxy only if circuit is not open
+        if (shouldTryProxy()) {
+            try {
+                historyRes = await proxyManager.fetch(historyUrl, requestOptions);
+                if (historyRes.ok) {
+                    historySuccess = true;
+                    recordProxyOutcome(true);
                 } else {
-                    // Circuit breaker open, skip proxy
-                }
-                
-                // Fallback to direct fetch if proxy failed or circuit is open
-                if (!historySuccess) {
-                    try {
-                        const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
-                        historyRes = await directFetch(historyUrl, requestOptions);
-                        if (historyRes.ok) {
-                            historySuccess = true;
-                        } else {
-                            throw new Error(`Direct fetch returned non-ok status: ${historyRes.status}`);
-                        }
-                    } catch (fetchErr) {
-                        logger.log('WARN', `[PI Update] Direct fetch also failed for history page ${page}`, fetchErr);
-                        hasMore = false;
-                        continue;
-                    }
+                    throw new Error(`Proxy returned non-ok status: ${historyRes.status}`);
                 }
-                
-                if (historySuccess && historyRes.ok) {
-                    const data = await historyRes.json();
-                    const trades = data.PublicHistoryPositions || [];
-                    
-                    if (trades.length > 0) {
-                        allHistoryTrades = allHistoryTrades.concat(trades);
-                        page++;
-                        hasMore = (trades.length === 1000); 
-                    } else {
-                        hasMore = false;
-                    }
+            } catch (proxyError) {
+                recordProxyOutcome(false);
+                logger.log('WARN', `[PI Update] Proxy failed for history (${getFailureCount()}/${getMaxFailures()}). Trying direct...`);
+            }
+        } else {
+            // Circuit breaker open, skip proxy
+        }
+        
+        // Fallback to direct fetch if proxy failed or circuit is open
+        if (!historySuccess) {
+            try {
+                const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
+                historyRes = await directFetch(historyUrl, requestOptions);
+                if (historyRes.ok) {
+                    historySuccess = true;
                 } else {
-                    logger.log('WARN', `[PI Update] History fetch failed on page ${page} for ${username}. Status: ${historyRes?.status || 'unknown'}`);
-                    hasMore = false;
+                    throw new Error(`Direct fetch returned non-ok status: ${historyRes.status}`);
                 }
-             } catch (e) {
-                 logger.log('WARN', `[PI Update] History fetch error page ${page}: ${e.message}`);
-                 hasMore = false;
-             }
+            } catch (fetchErr) {
+                logger.log('WARN', `[PI Update] Direct fetch also failed for history`, fetchErr);
+                historyRes = { ok: false, status: 500 };
+            }
         }
 
-        if (allHistoryTrades.length > 0) {
-             const historyPayload = { 
-                 PublicHistoryPositions: allHistoryTrades,
-                 fetchedAt: new Date()
-             };
-             await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyPayload, 'popular_investor');
-             logger.log('INFO', `[PI Update] Fetched total ${allHistoryTrades.length} trades for ${username}`);
+        if (historySuccess && historyRes.ok) {
+            const historyData = await historyRes.json();
+            const trades = historyData.PublicHistoryPositions || [];
+            
+            if (trades.length > 0) {
+                const historyPayload = { 
+                    PublicHistoryPositions: trades,
+                    fetchedAt: new Date()
+                };
+                await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyPayload, 'popular_investor');
+                logger.log('INFO', `[PI Update] Fetched ${trades.length} trades for ${username}`);
+            }
+        } else {
+            logger.log('WARN', `[PI Update] History fetch failed for ${username}. Status: ${historyRes?.status || 'unknown'}`);
         }
 
         logger.log('SUCCESS', `[PI Update] Completed full update for ${username}`);
@@ -389,11 +365,11 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
 
         await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
 
-        // History Fetch
+        // History Fetch - Single Page with Large ItemsPerPage
         const oneYearAgo = new Date();
         oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
         const historyBaseUrl = ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
-        const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=1000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
+        const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
 
         logger.log('INFO', `[On-Demand DEBUG] Fetching History URL: ${historyUrl}`);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.404",
+  "version": "1.0.406",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [