bulltrackers-module 1.0.577 → 1.0.579

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/functions/computation-system/context/ContextFactory.js +17 -8
  2. package/functions/computation-system/context/ManifestBuilder.js +5 -0
  3. package/functions/computation-system/data/AvailabilityChecker.js +111 -42
  4. package/functions/computation-system/data/CachedDataLoader.js +69 -34
  5. package/functions/computation-system/data/DependencyFetcher.js +38 -5
  6. package/functions/computation-system/executors/MetaExecutor.js +77 -7
  7. package/functions/computation-system/executors/StandardExecutor.js +68 -2
  8. package/package.json +1 -1
package/functions/computation-system/context/ContextFactory.js CHANGED
@@ -2,6 +2,7 @@
2
2
  * @fileoverview Factory for creating the Computation Context.
3
3
  * UPDATED: Injects verification and rankings data into context globally and locally.
4
4
  * UPDATED: Added support for historical ranking data in both Standard and Meta contexts.
5
+ * UPDATED: Added support for 'series' data (historical root data or computation results) in Global Data.
5
6
  */
6
7
  const mathLayer = require('../layers/index');
7
8
  const { LEGACY_MAPPING } = require('../topology/HashManager');
@@ -28,7 +29,9 @@ class ContextFactory {
28
29
  allVerifications,
29
30
  // [NEW] New Root Data Types for Profile Metrics
30
31
  ratings, pageViews, watchlistMembership, alertHistory,
31
- piMasterList, // [NEW]
32
+ piMasterList,
33
+ // [NEW] Series Data (Lookback for Root Data or Computation Results)
34
+ seriesData
32
35
  } = options;
33
36
 
34
37
  return {
@@ -58,7 +61,10 @@ class ContextFactory {
58
61
  pageViews: pageViews || {},
59
62
  watchlistMembership: watchlistMembership || {},
60
63
  alertHistory: alertHistory || {},
61
- piMasterList: piMasterList || {}, // [NEW]
64
+ piMasterList: piMasterList || {},
65
+ // [NEW] Expose Series Data
66
+ // Structure: { root: { [type]: { [date]: data } }, results: { [date]: { [calcName]: data } } }
67
+ series: seriesData || {}
62
68
  }
63
69
  };
64
70
  }
@@ -67,10 +73,12 @@ class ContextFactory {
67
73
  const {
68
74
  dateStr, metadata, mappings, insights, socialData, prices,
69
75
  computedDependencies, previousComputedDependencies, config, deps,
70
- allRankings, allRankingsYesterday, // [UPDATED] Accepted here
76
+ allRankings, allRankingsYesterday,
71
77
  allVerifications,
72
- // [NEW] New Root Data Types for Profile Metrics
73
- ratings, pageViews, watchlistMembership, alertHistory
78
+ // [NEW] New Root Data Types
79
+ ratings, pageViews, watchlistMembership, alertHistory,
80
+ // [NEW] Series Data
81
+ seriesData
74
82
  } = options;
75
83
 
76
84
  return {
@@ -85,13 +93,14 @@ class ContextFactory {
85
93
  meta: metadata, config, deps,
86
94
  globalData: {
87
95
  rankings: allRankings || [],
88
- rankingsYesterday: allRankingsYesterday || [], // [UPDATED] Injected here
96
+ rankingsYesterday: allRankingsYesterday || [],
89
97
  verifications: allVerifications || {},
90
- // [NEW] New Root Data Types for Profile Metrics
91
98
  ratings: ratings || {},
92
99
  pageViews: pageViews || {},
93
100
  watchlistMembership: watchlistMembership || {},
94
- alertHistory: alertHistory || {}
101
+ alertHistory: alertHistory || {},
102
+ // [NEW] Expose Series Data
103
+ series: seriesData || {}
95
104
  }
96
105
  };
97
106
  }
package/functions/computation-system/context/ManifestBuilder.js CHANGED
@@ -7,6 +7,7 @@
7
7
  /**
8
8
  * @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
9
9
  * UPDATED: Removed Automatic Infra Hashing. Now relies strictly on SYSTEM_EPOCH.
10
+ * UPDATED: Whitelisted 'rootDataSeries' and 'dependencySeries' metadata fields.
10
11
  */
11
12
  const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
12
13
  const { normalizeName } = require('../utils/utils');
@@ -253,6 +254,10 @@ function buildManifest(productLinesToRun = [], calculations) {
253
254
  isPage: metadata.isPage === true,
254
255
  isHistorical: metadata.isHistorical !== undefined ? metadata.isHistorical : false,
255
256
  rootDataDependencies: metadata.rootDataDependencies || [],
257
+ // [NEW] Pass Series Configuration
258
+ rootDataSeries: metadata.rootDataSeries || null,
259
+ dependencySeries: metadata.dependencySeries || null,
260
+
256
261
  canHaveMissingRoots: metadata.canHaveMissingRoots || false,
257
262
  userType: metadata.userType,
258
263
  dependencies: dependencies,
package/functions/computation-system/data/AvailabilityChecker.js CHANGED
@@ -3,6 +3,7 @@
3
3
  * REFACTORED: Fully supports granular flags for PI, Signed-In Users, Rankings, and Verification.
4
4
  * FIXED: Removed permissive fallbacks to enforce strict userType availability rules.
5
5
  * UPDATED: Added strict social data checking for Popular Investors and Signed-In Users.
6
+ * UPDATED: Added Optimistic Series Permission. Allows execution if "Today's" data is missing but a Lookback Series is requested.
6
7
  */
7
8
  const { normalizeName } = require('../utils/utils');
8
9
 
@@ -24,37 +25,49 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
24
25
  let isAvailable = false;
25
26
 
26
27
  if (dep === 'portfolio') {
27
- if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
28
- else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
29
- else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
30
- else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
31
- else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
32
- // REMOVED: Unconditional fallback that bypassed strict userType checks
28
+ if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
29
+ else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
30
+ else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
31
+ else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
32
+ else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
33
33
 
34
34
  if (!isAvailable) {
35
- if (userType === 'speculator') missing.push('speculatorPortfolio');
36
- else if (userType === 'normal') missing.push('normalPortfolio');
35
+ // [OPTIMIZATION] Optimistic Series Check
36
+ // If Today's Portfolio is missing, but the computation explicitly asks for a Series (History),
37
+ // we allow it to proceed optimistically, assuming historical data might exist.
38
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
39
+ available.push('portfolio');
40
+ continue;
41
+ }
42
+
43
+ if (userType === 'speculator') missing.push('speculatorPortfolio');
44
+ else if (userType === 'normal') missing.push('normalPortfolio');
37
45
  else if (userType === 'popular_investor') missing.push('piPortfolios');
38
- else if (userType === 'signed_in_user') missing.push('signedInUserPortfolio');
39
- else missing.push('portfolio');
46
+ else if (userType === 'signed_in_user') missing.push('signedInUserPortfolio');
47
+ else missing.push('portfolio');
40
48
  } else {
41
49
  available.push('portfolio');
42
50
  }
43
51
  }
44
- else if (dep === 'history') {
45
- if (userType === 'speculator' && rootDataStatus.speculatorHistory) isAvailable = true;
46
- else if (userType === 'normal' && rootDataStatus.normalHistory) isAvailable = true;
47
- else if (userType === 'popular_investor' && rootDataStatus.piHistory) isAvailable = true;
48
- else if (userType === 'signed_in_user' && rootDataStatus.signedInUserHistory) isAvailable = true;
49
- else if (userType === 'all' && rootDataStatus.hasHistory) isAvailable = true;
50
- // REMOVED: Unconditional fallback that bypassed strict userType checks
52
+ else if (dep === 'history') {
53
+ if (userType === 'speculator' && rootDataStatus.speculatorHistory) isAvailable = true;
54
+ else if (userType === 'normal' && rootDataStatus.normalHistory) isAvailable = true;
55
+ else if (userType === 'popular_investor' && rootDataStatus.piHistory) isAvailable = true;
56
+ else if (userType === 'signed_in_user' && rootDataStatus.signedInUserHistory) isAvailable = true;
57
+ else if (userType === 'all' && rootDataStatus.hasHistory) isAvailable = true;
51
58
 
52
59
  if (!isAvailable) {
53
- if (userType === 'speculator') missing.push('speculatorHistory');
54
- else if (userType === 'normal') missing.push('normalHistory');
60
+ // [OPTIMIZATION] Optimistic Series Check
61
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
62
+ available.push('history');
63
+ continue;
64
+ }
65
+
66
+ if (userType === 'speculator') missing.push('speculatorHistory');
67
+ else if (userType === 'normal') missing.push('normalHistory');
55
68
  else if (userType === 'popular_investor') missing.push('piHistory');
56
- else if (userType === 'signed_in_user') missing.push('signedInUserHistory');
57
- else missing.push('history');
69
+ else if (userType === 'signed_in_user') missing.push('signedInUserHistory');
70
+ else missing.push('history');
58
71
  } else {
59
72
  available.push('history');
60
73
  }
@@ -64,7 +77,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
64
77
  isAvailable = true;
65
78
  available.push('rankings');
66
79
  } else {
67
- missing.push('piRankings');
80
+ // [OPTIMIZATION] Optimistic Series Check
81
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
82
+ available.push('rankings');
83
+ } else {
84
+ missing.push('piRankings');
85
+ }
68
86
  }
69
87
  }
70
88
  else if (dep === 'verification') {
@@ -80,27 +98,35 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
80
98
  isAvailable = true;
81
99
  available.push('insights');
82
100
  } else {
83
- missing.push('insights');
101
+ // [OPTIMIZATION] Optimistic Series Check
102
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
103
+ available.push('insights');
104
+ } else {
105
+ missing.push('insights');
106
+ }
84
107
  }
85
108
  }
86
- // [UPDATED] Strict Social Data Checking
87
109
  else if (dep === 'social') {
88
- // Strictly check based on userType to prevent fallback to generic social data
89
110
  if (userType === 'popular_investor') {
90
- if (rootDataStatus.hasPISocial) isAvailable = true;
111
+ if (rootDataStatus.hasPISocial) isAvailable = true;
91
112
  } else if (userType === 'signed_in_user') {
92
- if (rootDataStatus.hasSignedInSocial) isAvailable = true;
113
+ if (rootDataStatus.hasSignedInSocial) isAvailable = true;
93
114
  } else {
94
- // For 'all', 'normal', 'speculator' or undefined, accept generic social data
95
- if (rootDataStatus.hasSocial) isAvailable = true;
115
+ if (rootDataStatus.hasSocial) isAvailable = true;
96
116
  }
97
117
 
98
118
  if (isAvailable) {
99
119
  available.push('social');
100
120
  } else {
101
- if (userType === 'popular_investor') missing.push('piSocial');
121
+ // [OPTIMIZATION] Optimistic Series Check
122
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
123
+ available.push('social');
124
+ continue;
125
+ }
126
+
127
+ if (userType === 'popular_investor') missing.push('piSocial');
102
128
  else if (userType === 'signed_in_user') missing.push('signedInSocial');
103
- else missing.push('social');
129
+ else missing.push('social');
104
130
  }
105
131
  }
106
132
  else if (dep === 'price') {
@@ -108,7 +134,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
108
134
  isAvailable = true;
109
135
  available.push('price');
110
136
  } else {
111
- missing.push('price');
137
+ // [OPTIMIZATION] Optimistic Series Check
138
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
139
+ available.push('price');
140
+ } else {
141
+ missing.push('price');
142
+ }
112
143
  }
113
144
  }
114
145
  // [NEW] New Root Data Types for Profile Metrics
@@ -117,7 +148,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
117
148
  isAvailable = true;
118
149
  available.push('ratings');
119
150
  } else {
120
- missing.push('piRatings');
151
+ // [OPTIMIZATION] Optimistic Series Check (Vital for sparse data like ratings)
152
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
153
+ available.push('ratings');
154
+ } else {
155
+ missing.push('piRatings');
156
+ }
121
157
  }
122
158
  }
123
159
  else if (dep === 'pageViews') {
@@ -125,7 +161,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
125
161
  isAvailable = true;
126
162
  available.push('pageViews');
127
163
  } else {
128
- missing.push('piPageViews');
164
+ // [OPTIMIZATION] Optimistic Series Check
165
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
166
+ available.push('pageViews');
167
+ } else {
168
+ missing.push('piPageViews');
169
+ }
129
170
  }
130
171
  }
131
172
  else if (dep === 'watchlist') {
@@ -133,7 +174,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
133
174
  isAvailable = true;
134
175
  available.push('watchlist');
135
176
  } else {
136
- missing.push('watchlistMembership');
177
+ // [OPTIMIZATION] Optimistic Series Check
178
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
179
+ available.push('watchlist');
180
+ } else {
181
+ missing.push('watchlistMembership');
182
+ }
137
183
  }
138
184
  }
139
185
  else if (dep === 'alerts') {
@@ -141,7 +187,14 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
141
187
  isAvailable = true;
142
188
  available.push('alerts');
143
189
  } else {
144
- missing.push('piAlertHistory');
190
+ // [OPTIMIZATION] Optimistic Series Check
191
+ // This explicitly solves the Alert History edge case where today's snapshot
192
+ // might be missing but the lookback period contains data.
193
+ if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
194
+ available.push('alerts');
195
+ } else {
196
+ missing.push('piAlertHistory');
197
+ }
145
198
  }
146
199
  }
147
200
  }
@@ -244,12 +297,28 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
244
297
  logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
245
298
  return {
246
299
  status: {
247
- hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false,
248
- speculatorPortfolio: false, normalPortfolio: false, speculatorHistory: false, normalHistory: false,
249
- piRankings: false, piPortfolios: false, piDeepPortfolios: false, piHistory: false,
250
- signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false,
251
- hasPISocial: false, hasSignedInSocial: false,
252
- piRatings: false, piPageViews: false, watchlistMembership: false, piAlertHistory: false
300
+ hasPortfolio: false,
301
+ hasHistory: false,
302
+ hasSocial: false,
303
+ hasInsights: false,
304
+ hasPrices: false,
305
+ speculatorPortfolio: false,
306
+ normalPortfolio: false,
307
+ speculatorHistory: false,
308
+ normalHistory: false,
309
+ piRankings: false,
310
+ piPortfolios: false,
311
+ piDeepPortfolios: false,
312
+ piHistory: false,
313
+ signedInUserPortfolio: false,
314
+ signedInUserHistory: false,
315
+ signedInUserVerification: false,
316
+ hasPISocial: false,
317
+ hasSignedInSocial: false,
318
+ piRatings: false,
319
+ piPageViews: false,
320
+ watchlistMembership: false,
321
+ piAlertHistory: false
253
322
  }
254
323
  };
255
324
  }
package/functions/computation-system/data/CachedDataLoader.js CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * {
3
3
  * type: uploaded file
4
- * fileName: bulltrackers-module/functions/computation-system/data/CachedDataLoader.js
4
+ * fileName: computation-system/data/CachedDataLoader.js
5
5
  * }
6
6
  */
7
7
  const {
@@ -9,14 +9,14 @@ const {
9
9
  loadDailySocialPostInsights,
10
10
  getRelevantShardRefs,
11
11
  getPriceShardRefs,
12
- loadVerificationProfiles, // [NEW]
13
- loadPopularInvestorRankings, // [NEW]
14
- loadPIRatings, // [NEW]
15
- loadPIPageViews, // [NEW]
16
- loadWatchlistMembership: loadWatchlistMembershipData, // [NEW] Renamed to avoid conflict
17
- loadPIAlertHistory, // [NEW]
18
- loadPIWatchlistData, // [NEW] PI-centric watchlist data
19
- loadPopularInvestorMasterList // [NEW]
12
+ loadVerificationProfiles,
13
+ loadPopularInvestorRankings,
14
+ loadPIRatings,
15
+ loadPIPageViews,
16
+ loadWatchlistMembership: loadWatchlistMembershipData,
17
+ loadPIAlertHistory,
18
+ loadPIWatchlistData,
19
+ loadPopularInvestorMasterList
20
20
  } = require('../utils/data_loader');
21
21
  const zlib = require('zlib');
22
22
 
@@ -28,14 +28,14 @@ class CachedDataLoader {
28
28
  mappings: null,
29
29
  insights: new Map(),
30
30
  social: new Map(),
31
- verifications: null, // [NEW]
32
- rankings: new Map(), // [NEW]
33
- ratings: new Map(), // [NEW]
34
- pageViews: new Map(), // [NEW]
35
- watchlistMembership: new Map(), // [NEW]
36
- alertHistory: new Map(),// [NEW]
37
- piWatchlistData: new Map(), // [NEW] PI-centric watchlist data cache (keyed by piCid)
38
- piMasterList: null // [NEW] Singleton cache (not date dependent)
31
+ verifications: null,
32
+ rankings: new Map(),
33
+ ratings: new Map(),
34
+ pageViews: new Map(),
35
+ watchlistMembership: new Map(),
36
+ alertHistory: new Map(),
37
+ piWatchlistData: new Map(),
38
+ piMasterList: null
39
39
  };
40
40
  }
41
41
 
@@ -51,6 +51,7 @@ class CachedDataLoader {
51
51
  return data;
52
52
  }
53
53
 
54
+ // ... [Existing load methods: loadMappings, loadInsights, etc. unchanged] ...
54
55
  async loadMappings() {
55
56
  if (this.cache.mappings) return this.cache.mappings;
56
57
  const { calculationUtils } = this.deps;
@@ -58,7 +59,6 @@ class CachedDataLoader {
58
59
  return this.cache.mappings;
59
60
  }
60
61
 
61
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
62
62
  async loadInsights(dateStr) {
63
63
  if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr);
64
64
  const promise = loadDailyInsights(this.config, this.deps, dateStr);
@@ -66,7 +66,6 @@ class CachedDataLoader {
66
66
  return promise;
67
67
  }
68
68
 
69
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
70
69
  async loadSocial(dateStr) {
71
70
  if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr);
72
71
  const promise = loadDailySocialPostInsights(this.config, this.deps, dateStr);
@@ -74,7 +73,6 @@ class CachedDataLoader {
74
73
  return promise;
75
74
  }
76
75
 
77
- // [NEW]
78
76
  async loadVerifications() {
79
77
  if (this.cache.verifications) return this.cache.verifications;
80
78
  const verifications = await loadVerificationProfiles(this.config, this.deps);
@@ -82,8 +80,6 @@ class CachedDataLoader {
82
80
  return verifications;
83
81
  }
84
82
 
85
- // [NEW]
86
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
87
83
  async loadRankings(dateStr) {
88
84
  if (this.cache.rankings.has(dateStr)) return this.cache.rankings.get(dateStr);
89
85
  const promise = loadPopularInvestorRankings(this.config, this.deps, dateStr);
@@ -91,8 +87,6 @@ class CachedDataLoader {
91
87
  return promise;
92
88
  }
93
89
 
94
- // [NEW] Load PI Ratings Data
95
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
96
90
  async loadRatings(dateStr) {
97
91
  if (this.cache.ratings.has(dateStr)) return this.cache.ratings.get(dateStr);
98
92
  const promise = loadPIRatings(this.config, this.deps, dateStr);
@@ -100,8 +94,6 @@ class CachedDataLoader {
100
94
  return promise;
101
95
  }
102
96
 
103
- // [NEW] Load PI Page Views Data
104
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
105
97
  async loadPageViews(dateStr) {
106
98
  if (this.cache.pageViews.has(dateStr)) return this.cache.pageViews.get(dateStr);
107
99
  const promise = loadPIPageViews(this.config, this.deps, dateStr);
@@ -109,8 +101,6 @@ class CachedDataLoader {
109
101
  return promise;
110
102
  }
111
103
 
112
- // [NEW] Load Watchlist Membership Data
113
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
114
104
  async loadWatchlistMembership(dateStr) {
115
105
  if (this.cache.watchlistMembership.has(dateStr)) return this.cache.watchlistMembership.get(dateStr);
116
106
  const promise = loadWatchlistMembershipData(this.config, this.deps, dateStr);
@@ -118,8 +108,6 @@ class CachedDataLoader {
118
108
  return promise;
119
109
  }
120
110
 
121
- // [NEW] Load PI Alert History Data
122
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
123
111
  async loadAlertHistory(dateStr) {
124
112
  if (this.cache.alertHistory.has(dateStr)) return this.cache.alertHistory.get(dateStr);
125
113
  const promise = loadPIAlertHistory(this.config, this.deps, dateStr);
@@ -127,9 +115,6 @@ class CachedDataLoader {
127
115
  return promise;
128
116
  }
129
117
 
130
- // [NEW] Load PI-Centric Watchlist Data
131
- // Loads watchlist data from PopularInvestors/{piCid}/watchlistData/current
132
- // [FIX] Cache promises to prevent race conditions when multiple users request same data
133
118
  async loadPIWatchlistData(piCid) {
134
119
  const piCidStr = String(piCid);
135
120
  if (this.cache.piWatchlistData.has(piCidStr)) return this.cache.piWatchlistData.get(piCidStr);
@@ -156,13 +141,63 @@ class CachedDataLoader {
156
141
  return {};
157
142
  }
158
143
  }
159
- // [NEW] Load PI Master List (Global, Cached)
144
+
160
145
  async loadPIMasterList() {
161
146
  if (this.cache.piMasterList) return this.cache.piMasterList;
162
147
  const data = await loadPopularInvestorMasterList(this.config, this.deps);
163
148
  this.cache.piMasterList = data;
164
149
  return data;
165
150
  }
151
+
152
+ // --- [NEW] Series Loading Logic ---
153
+ /**
154
+ * Optimistically loads a series of root data over a lookback period.
155
+ * @param {string} loaderMethod - The method name to call (e.g., 'loadAlertHistory')
156
+ * @param {string} dateStr - The end date (exclusive or inclusive depending on data availability)
157
+ * @param {number} lookbackDays - Number of days to look back
158
+ */
159
+ async loadSeries(loaderMethod, dateStr, lookbackDays) {
160
+ if (!this[loaderMethod]) throw new Error(`[CachedDataLoader] Unknown method ${loaderMethod}`);
161
+
162
+ const results = {};
163
+ const endDate = new Date(dateStr);
164
+ const promises = [];
165
+
166
+ // Fetch N days back (including dateStr if relevant, usually handled by caller logic)
167
+ // Here we fetch [dateStr, dateStr-1, ... dateStr-(N-1)]
168
+ for (let i = 0; i < lookbackDays; i++) {
169
+ const d = new Date(endDate);
170
+ d.setUTCDate(d.getUTCDate() - i);
171
+ const dString = d.toISOString().slice(0, 10);
172
+
173
+ promises.push(
174
+ this[loaderMethod](dString)
175
+ .then(data => ({ date: dString, data }))
176
+ .catch(err => {
177
+ // Optimistic: Log warning but continue
178
+ console.warn(`[CachedDataLoader] Failed to load series item ${loaderMethod} for ${dString}: ${err.message}`);
179
+ return { date: dString, data: null };
180
+ })
181
+ );
182
+ }
183
+
184
+ const loaded = await Promise.all(promises);
185
+
186
+ let foundCount = 0;
187
+ loaded.forEach(({ date, data }) => {
188
+ if (data) {
189
+ results[date] = data;
190
+ foundCount++;
191
+ }
192
+ });
193
+
194
+ return {
195
+ dates: Object.keys(results).sort(),
196
+ data: results,
197
+ found: foundCount,
198
+ requested: lookbackDays
199
+ };
200
+ }
166
201
  }
167
202
 
168
203
  module.exports = { CachedDataLoader };
package/functions/computation-system/data/DependencyFetcher.js CHANGED
@@ -2,9 +2,10 @@
2
2
  * @fileoverview Fetches results from previous computations, handling auto-sharding and decompression.
3
3
  */
4
4
  const { normalizeName } = require('../utils/utils');
5
- const zlib = require('zlib'); // [NEW]
5
+ const zlib = require('zlib');
6
6
 
7
7
  async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
8
+ // ... [Existing implementation unchanged] ...
8
9
  const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
9
10
  const calcsToFetch = new Set();
10
11
 
@@ -41,10 +42,8 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
41
42
  if (!doc.exists) return;
42
43
  const data = doc.data();
43
44
 
44
- // --- [NEW] DECOMPRESSION LOGIC ---
45
45
  if (data._compressed === true && data.payload) {
46
46
  try {
47
- // Firestore returns Buffers automatically
48
47
  const unzipped = zlib.gunzipSync(data.payload);
49
48
  fetched[name] = JSON.parse(unzipped.toString());
50
49
  } catch (e) {
@@ -52,7 +51,6 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
52
51
  fetched[name] = {};
53
52
  }
54
53
  }
55
- // --- END NEW LOGIC ---
56
54
  else if (data._sharded === true) {
57
55
  hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name));
58
56
  } else if (data._completed) {
@@ -81,4 +79,39 @@ async function hydrateAutoShardedResult(docRef, resultName) {
81
79
  return { name: resultName, data: assembledData };
82
80
  }
83
81
 
84
- module.exports = { fetchExistingResults };
82
+ // [NEW] Fetch Result Series
83
+ async function fetchResultSeries(dateStr, calcsToFetchNames, fullManifest, config, deps, lookbackDays) {
84
+ const results = {}; // Structure: { [date]: { [calcName]: data } }
85
+ const endDate = new Date(dateStr);
86
+ const promises = [];
87
+
88
+ // Create a dummy "calcsInPass" object to satisfy fetchExistingResults signature
89
+ // We just need objects that have .dependencies matching what we want to fetch
90
+ const dummyCalc = { dependencies: calcsToFetchNames, isHistorical: false };
91
+
92
+ for (let i = 0; i < lookbackDays; i++) {
93
+ const d = new Date(endDate);
94
+ d.setUTCDate(d.getUTCDate() - i);
95
+ const dString = d.toISOString().slice(0, 10);
96
+
97
+ promises.push(
98
+ fetchExistingResults(dString, [dummyCalc], fullManifest, config, deps, false)
99
+ .then(res => ({ date: dString, data: res }))
100
+ .catch(e => {
101
+ console.warn(`[DependencyFetcher] Failed to fetch series for ${dString}: ${e.message}`);
102
+ return { date: dString, data: {} };
103
+ })
104
+ );
105
+ }
106
+
107
+ const series = await Promise.all(promises);
108
+ series.forEach(({ date, data }) => {
109
+ if (data && Object.keys(data).length > 0) {
110
+ results[date] = data;
111
+ }
112
+ });
113
+
114
+ return results;
115
+ }
116
+
117
+ module.exports = { fetchExistingResults, fetchResultSeries };
package/functions/computation-system/executors/MetaExecutor.js CHANGED
@@ -4,11 +4,13 @@
4
4
  * UPDATED: Tracks processed shard/item counts.
5
5
  * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
6
6
  * UPDATED: Support for historical rankings in Meta Context.
7
+ * UPDATED: Added support for loading Series Data (Root & Results) for lookbacks.
7
8
  */
8
- const { normalizeName } = require('../utils/utils');
9
- const { CachedDataLoader } = require('../data/CachedDataLoader');
10
- const { ContextFactory } = require('../context/ContextFactory');
11
- const { commitResults } = require('../persistence/ResultCommitter');
9
+ const { normalizeName } = require('../utils/utils');
10
+ const { CachedDataLoader } = require('../data/CachedDataLoader');
11
+ const { ContextFactory } = require('../context/ContextFactory');
12
+ const { commitResults } = require('../persistence/ResultCommitter');
13
+ const { fetchResultSeries } = require('../data/DependencyFetcher'); // [NEW] Import series fetcher
12
14
 
13
15
  class MetaExecutor {
14
16
  static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
@@ -125,6 +127,64 @@ class MetaExecutor {
125
127
  }
126
128
  }
127
129
 
130
+ // --- [NEW] Series / Lookback Loading Logic ---
131
+ const rootSeriesRequests = {};
132
+ const dependencySeriesRequests = {};
133
+
134
+ // 1. Identify requirements from manifests
135
+ calcs.forEach(c => {
136
+ // Check for Root Data Series
137
+ if (c.rootDataSeries) {
138
+ Object.entries(c.rootDataSeries).forEach(([type, conf]) => {
139
+ const days = typeof conf === 'object' ? conf.lookback : conf;
140
+ if (!rootSeriesRequests[type] || days > rootSeriesRequests[type]) {
141
+ rootSeriesRequests[type] = days;
142
+ }
143
+ });
144
+ }
145
+ // Check for Computation Result Series
146
+ if (c.dependencySeries) {
147
+ Object.entries(c.dependencySeries).forEach(([depName, conf]) => {
148
+ const days = typeof conf === 'object' ? conf.lookback : conf;
149
+ const normalized = normalizeName(depName);
150
+ if (!dependencySeriesRequests[normalized] || days > dependencySeriesRequests[normalized]) {
151
+ dependencySeriesRequests[normalized] = days;
152
+ }
153
+ });
154
+ }
155
+ });
156
+
157
+ const seriesData = { root: {}, results: {} };
158
+
159
+ // 2. Load Root Series
160
+ for (const [type, days] of Object.entries(rootSeriesRequests)) {
161
+ let loaderMethod = null;
162
+ if (type === 'alerts') loaderMethod = 'loadAlertHistory';
163
+ else if (type === 'insights') loaderMethod = 'loadInsights';
164
+ else if (type === 'ratings') loaderMethod = 'loadRatings';
165
+ else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
166
+ // Add other root types if needed
167
+
168
+ if (loaderMethod) {
169
+ logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
170
+ // Assume CachedDataLoader has loadSeries method (added in previous step)
171
+ const series = await loader.loadSeries(loaderMethod, dStr, days);
172
+ seriesData.root[type] = series.data; // map of date -> data
173
+ }
174
+ }
175
+
176
+ // 3. Load Computation Result Series
177
+ const calcNamesToFetch = Object.keys(dependencySeriesRequests);
178
+ if (calcNamesToFetch.length > 0) {
179
+ const maxDays = Math.max(...Object.values(dependencySeriesRequests));
180
+ logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
181
+
182
+ // We pass the list of manifests (calcs) so the fetcher knows details if needed
183
+ const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, calcs, config, deps, maxDays);
184
+ seriesData.results = resultsSeries; // map of date -> { calcName: data }
185
+ }
186
+ // ---------------------------------------------
187
+
128
188
  const state = {};
129
189
  for (const c of calcs) {
130
190
  const inst = new c.class();
@@ -146,7 +206,9 @@ class MetaExecutor {
146
206
  ratings: ratings || {},
147
207
  pageViews: pageViews || {},
148
208
  watchlistMembership: watchlistMembership || {},
149
- alertHistory: alertHistory || {}
209
+ alertHistory: alertHistory || {},
210
+ // [NEW] Pass Series Data
211
+ seriesData
150
212
  });
151
213
 
152
214
  try {
@@ -239,6 +301,10 @@ class MetaExecutor {
239
301
  }
240
302
  }
241
303
 
304
+ // [NOTE] "executeOncePerDay" is typically for sharded price/batch jobs.
305
+ // We initialize empty series data to maintain compatibility.
306
+ const seriesData = { root: {}, results: {} };
307
+
242
308
  if (metadata.rootDataDependencies?.includes('price')) {
243
309
  logger.log('INFO', `[Executor] Running Batched/Sharded Execution for ${metadata.name}`);
244
310
  const shardRefs = await loader.getPriceShardReferences();
@@ -257,7 +323,9 @@ class MetaExecutor {
257
323
  ratings: ratings || {},
258
324
  pageViews: pageViews || {},
259
325
  watchlistMembership: watchlistMembership || {},
260
- alertHistory: alertHistory || {}
326
+ alertHistory: alertHistory || {},
327
+ // [NEW] Pass Series Data
328
+ seriesData
261
329
  });
262
330
 
263
331
  await calcInstance.process(partialContext);
@@ -282,7 +350,9 @@ class MetaExecutor {
282
350
  ratings: ratings || {},
283
351
  pageViews: pageViews || {},
284
352
  watchlistMembership: watchlistMembership || {},
285
- alertHistory: alertHistory || {}
353
+ alertHistory: alertHistory || {},
354
+ // [NEW] Pass Series Data
355
+ seriesData
286
356
  });
287
357
  const res = await calcInstance.process(context);
288
358
 
package/functions/computation-system/executors/StandardExecutor.js CHANGED
@@ -9,6 +9,8 @@ const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistory
9
9
  const { CachedDataLoader } = require('../data/CachedDataLoader');
10
10
  const { ContextFactory } = require('../context/ContextFactory');
11
11
  const { commitResults } = require('../persistence/ResultCommitter');
12
+ // [NEW] Import series fetcher for computation results
13
+ const { fetchResultSeries } = require('../data/DependencyFetcher');
12
14
  const mathLayer = require('../layers/index');
13
15
  const { performance } = require('perf_hooks');
14
16
  const v8 = require('v8');
@@ -124,6 +126,66 @@ class StandardExecutor {
124
126
  const setupDuration = performance.now() - startSetup;
125
127
  Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
126
128
 
129
+ // --- [NEW] Series / Lookback Loading Logic ---
130
+ const rootSeriesRequests = {};
131
+ const dependencySeriesRequests = {};
132
+
133
+ // 1. Identify requirements from manifests
134
+ streamingCalcs.forEach(c => {
135
+ // Check for Root Data Series (e.g., { insights: 7, alerts: 30 })
136
+ if (c.manifest.rootDataSeries) {
137
+ Object.entries(c.manifest.rootDataSeries).forEach(([type, conf]) => {
138
+ const days = typeof conf === 'object' ? conf.lookback : conf;
139
+ if (!rootSeriesRequests[type] || days > rootSeriesRequests[type]) {
140
+ rootSeriesRequests[type] = days;
141
+ }
142
+ });
143
+ }
144
+ // Check for Computation Result Series (e.g., { 'RiskScore': 7 })
145
+ if (c.manifest.dependencySeries) {
146
+ Object.entries(c.manifest.dependencySeries).forEach(([depName, conf]) => {
147
+ const days = typeof conf === 'object' ? conf.lookback : conf;
148
+ const normalized = normalizeName(depName);
149
+ if (!dependencySeriesRequests[normalized] || days > dependencySeriesRequests[normalized]) {
150
+ dependencySeriesRequests[normalized] = days;
151
+ }
152
+ });
153
+ }
154
+ });
155
+
156
+ const seriesData = { root: {}, results: {} };
157
+
158
+ // 2. Load Root Series
159
+ for (const [type, days] of Object.entries(rootSeriesRequests)) {
160
+ let loaderMethod = null;
161
+ if (type === 'alerts') loaderMethod = 'loadAlertHistory';
162
+ else if (type === 'insights') loaderMethod = 'loadInsights';
163
+ else if (type === 'ratings') loaderMethod = 'loadRatings';
164
+ else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
165
+ // Add other root types as needed...
166
+
167
+ if (loaderMethod) {
168
+ logger.log('INFO', `[StandardExecutor] Loading ${days}-day series for Root Data '${type}'...`);
169
+ // Assume CachedDataLoader has loadSeries method (added in previous step)
170
+ const series = await cachedLoader.loadSeries(loaderMethod, dateStr, days);
171
+ seriesData.root[type] = series.data; // map of date -> data
172
+ }
173
+ }
174
+
175
+ // 3. Load Computation Result Series
176
+ const calcNamesToFetch = Object.keys(dependencySeriesRequests);
177
+ if (calcNamesToFetch.length > 0) {
178
+ const maxDays = Math.max(...Object.values(dependencySeriesRequests));
179
+ logger.log('INFO', `[StandardExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
180
+
181
+ // We pass the full list of manifests so the fetcher knows where to look
182
+ const allManifests = streamingCalcs.map(c => c.manifest);
183
+ const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, allManifests, config, deps, maxDays);
184
+
185
+ seriesData.results = resultsSeries; // map of date -> { calcName: data }
186
+ }
187
+ // ---------------------------------------------
188
+
127
189
  const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
128
190
  const prevDateStr = prevDate.toISOString().slice(0, 10);
129
191
 
@@ -158,7 +220,9 @@ class StandardExecutor {
158
220
  calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk,
159
221
  fetchedDeps, previousFetchedDeps, config, deps, cachedLoader,
160
222
  executionStats[normalizeName(calc.manifest.name)],
161
- earliestDates
223
+ earliestDates,
224
+ // [NEW] Pass loaded series data
225
+ seriesData
162
226
  )
163
227
  ));
164
228
 
@@ -261,7 +325,7 @@ class StandardExecutor {
261
325
  if (newResult.failureReport) failureAcc.push(...newResult.failureReport);
262
326
  }
263
327
 
264
- static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates) {
328
+ static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates, seriesData = {}) {
265
329
  const { logger } = deps;
266
330
  const targetUserType = metadata.userType;
267
331
  // [FIX] Always load Global Helpers
@@ -426,6 +490,8 @@ class StandardExecutor {
426
490
  alertHistory: alertHistory || {},
427
491
 
428
492
  piMasterList,
493
+ // [NEW] Pass Series Data
494
+ seriesData
429
495
  });
430
496
 
431
497
  if (metadata.requiresEarliestDataDate && earliestDates) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bulltrackers-module",
3
- "version": "1.0.577",
3
+ "version": "1.0.579",
4
4
  "description": "Helper Functions for Bulltrackers.",
5
5
  "main": "index.js",
6
6
  "files": [