bulltrackers-module 1.0.631 → 1.0.633

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,17 +20,32 @@ class ContextFactory {
20
20
 
21
21
  static buildPerUserContext(options) {
22
22
  const {
23
- todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory,
24
- userId, userType, dateStr, metadata, mappings, insights, socialData,
25
- computedDependencies, previousComputedDependencies, config, deps,
23
+ todayPortfolio,
24
+ yesterdayPortfolio,
25
+ todayHistory,
26
+ yesterdayHistory,
27
+ userId,
28
+ userType,
29
+ dateStr,
30
+ metadata,
31
+ mappings,
32
+ insights,
33
+ socialData,
34
+ computedDependencies,
35
+ previousComputedDependencies,
36
+ config,
37
+ deps,
26
38
  verification,
27
- rankings, yesterdayRankings, // User-specific rank entries
28
- allRankings, allRankingsYesterday, // Global rank lists
39
+ rankings,
40
+ yesterdayRankings,
41
+ allRankings,
42
+ allRankingsYesterday,
29
43
  allVerifications,
30
- // [NEW] New Root Data Types for Profile Metrics
31
- ratings, pageViews, watchlistMembership, alertHistory,
44
+ ratings,
45
+ pageViews,
46
+ watchlistMembership,
47
+ alertHistory,
32
48
  piMasterList,
33
- // [NEW] Series Data (Lookback for Root Data or Computation Results)
34
49
  seriesData
35
50
  } = options;
36
51
 
@@ -56,14 +71,11 @@ class ContextFactory {
56
71
  rankings: allRankings || [],
57
72
  rankingsYesterday: allRankingsYesterday || [],
58
73
  verifications: allVerifications || {},
59
- // [NEW] New Root Data Types for Profile Metrics
60
74
  ratings: ratings || {},
61
75
  pageViews: pageViews || {},
62
76
  watchlistMembership: watchlistMembership || {},
63
77
  alertHistory: alertHistory || {},
64
78
  piMasterList: piMasterList || {},
65
- // [NEW] Expose Series Data
66
- // Structure: { root: { [type]: { [date]: data } }, results: { [date]: { [calcName]: data } } }
67
79
  series: seriesData || {}
68
80
  }
69
81
  };
@@ -71,13 +83,23 @@ class ContextFactory {
71
83
 
72
84
  static buildMetaContext(options) {
73
85
  const {
74
- dateStr, metadata, mappings, insights, socialData, prices,
75
- computedDependencies, previousComputedDependencies, config, deps,
76
- allRankings, allRankingsYesterday,
86
+ dateStr,
87
+ metadata,
88
+ mappings,
89
+ insights,
90
+ socialData,
91
+ prices,
92
+ computedDependencies,
93
+ previousComputedDependencies,
94
+ config,
95
+ deps,
96
+ allRankings,
97
+ allRankingsYesterday,
77
98
  allVerifications,
78
- // [NEW] New Root Data Types
79
- ratings, pageViews, watchlistMembership, alertHistory,
80
- // [NEW] Series Data
99
+ ratings,
100
+ pageViews,
101
+ watchlistMembership,
102
+ alertHistory,
81
103
  seriesData
82
104
  } = options;
83
105
 
@@ -99,7 +121,6 @@ class ContextFactory {
99
121
  pageViews: pageViews || {},
100
122
  watchlistMembership: watchlistMembership || {},
101
123
  alertHistory: alertHistory || {},
102
- // [NEW] Expose Series Data
103
124
  series: seriesData || {}
104
125
  }
105
126
  };
@@ -30,11 +30,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
30
30
  let isAvailable = false;
31
31
 
32
32
  if (dep === 'portfolio') {
33
- if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
34
- else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
33
+ if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
34
+ else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
35
35
  else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
36
- else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
37
- else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
36
+ else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
37
+ else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
38
38
 
39
39
  if (!isAvailable) {
40
40
  // [OPTIMIZATION] Optimistic Series Check
@@ -284,12 +284,28 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
284
284
  logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
285
285
  return {
286
286
  status: {
287
- hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false,
288
- speculatorPortfolio: false, normalPortfolio: false, speculatorHistory: false, normalHistory: false,
289
- piRankings: false, piPortfolios: false, piDeepPortfolios: false, piHistory: false,
290
- signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false,
291
- hasPISocial: false, hasSignedInSocial: false,
292
- piRatings: false, piPageViews: false, watchlistMembership: false, piAlertHistory: false
287
+ hasPortfolio: false,
288
+ hasHistory: false,
289
+ hasSocial: false,
290
+ hasInsights: false,
291
+ hasPrices: false,
292
+ speculatorPortfolio: false,
293
+ normalPortfolio: false,
294
+ speculatorHistory: false,
295
+ normalHistory: false,
296
+ piRankings: false,
297
+ piPortfolios: false,
298
+ piDeepPortfolios: false,
299
+ piHistory: false,
300
+ signedInUserPortfolio: false,
301
+ signedInUserHistory: false,
302
+ signedInUserVerification: false,
303
+ hasPISocial: false,
304
+ hasSignedInSocial: false,
305
+ piRatings: false,
306
+ piPageViews: false,
307
+ watchlistMembership: false,
308
+ piAlertHistory: false
293
309
  }
294
310
  };
295
311
  }
@@ -361,5 +377,5 @@ module.exports = {
361
377
  checkRootDependencies,
362
378
  checkRootDataAvailability,
363
379
  getViableCalculations,
364
- getAvailabilityWindow // [NEW] Exported
380
+ getAvailabilityWindow
365
381
  };
@@ -1,206 +1,153 @@
1
1
  /**
2
- * @fileoverview Fetches results from previous computations, handling auto-sharding and decompression.
3
- * UPDATED: Implemented 'Batched Series Fetching' to reduce Firestore read operations by ~98% for time-series lookups.
2
+ * @fileoverview Fetches dependencies for computations.
3
+ * UPDATED: Uses 'manifestLookup' to resolve the correct category (Core vs Non-Core).
4
+ * UPDATED: Supports automatic reassembly of sharded results (_shards subcollection).
5
+ * UPDATED: Supports decompression of zipped results.
4
6
  */
5
7
  const { normalizeName } = require('../utils/utils');
6
- const zlib = require('zlib');
7
- const pLimit = require('p-limit');
8
+ const zlib = require('zlib');
8
9
 
9
- async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
10
- const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
11
- const calcsToFetch = new Set();
10
+ /**
11
+ * Fetches dependencies for a specific date (Standard pass).
12
+ * @param {Date} date - The target date.
13
+ * @param {Array} calcs - The computations requiring dependencies.
14
+ * @param {Object} config - System config.
15
+ * @param {Object} deps - System dependencies (db, logger).
16
+ * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
17
+ */
18
+ async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}) {
19
+ const { db, logger } = deps;
20
+ const dStr = date.toISOString().slice(0, 10);
12
21
 
13
- for (const calc of calcsInPass) {
14
- if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d)));
15
- if (includeSelf && calc.isHistorical) calcsToFetch.add(normalizeName(calc.name));
16
- }
22
+ // 1. Identify unique dependencies needed
23
+ const needed = new Set();
24
+ calcs.forEach(c => {
25
+ if (c.getDependencies) {
26
+ const reqs = c.getDependencies();
27
+ reqs.forEach(r => needed.add(normalizeName(r)));
28
+ }
29
+ });
17
30
 
18
- if (!calcsToFetch.size) return {};
31
+ if (needed.size === 0) return {};
19
32
 
20
- const fetched = {};
21
- const docRefs = [];
22
- const names = [];
23
-
24
- for (const name of calcsToFetch) {
25
- const m = manifestMap.get(name);
26
- if (m) {
27
- docRefs.push(db.collection(config.resultsCollection)
28
- .doc(dateStr)
29
- .collection(config.resultsSubcollection)
30
- .doc(m.category || 'unknown')
31
- .collection(config.computationsSubcollection)
32
- .doc(name));
33
- names.push(name);
34
- }
35
- }
36
-
37
- if (docRefs.length) {
38
- const snaps = await db.getAll(...docRefs);
39
- const hydrationPromises = [];
40
-
41
- snaps.forEach((doc, i) => {
42
- const name = names[i];
43
- if (!doc.exists) return;
44
- const data = doc.data();
45
-
46
- // Handle Decompression
47
- if (data._compressed === true && data.payload) {
48
- try {
49
- const unzipped = zlib.gunzipSync(data.payload);
50
- fetched[name] = JSON.parse(unzipped.toString());
51
- } catch (e) {
52
- console.error(`[Hydration] Failed to decompress ${name}:`, e);
53
- fetched[name] = {};
54
- }
55
- }
56
- // Handle Sharding
57
- else if (data._sharded === true) {
58
- hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name));
59
- }
60
- // Standard
61
- else if (data._completed) {
62
- fetched[name] = data;
63
- }
64
- });
65
-
66
- if (hydrationPromises.length > 0) {
67
- const hydratedResults = await Promise.all(hydrationPromises);
68
- hydratedResults.forEach(res => { fetched[res.name] = res.data; });
69
- }
70
- }
71
- return fetched;
72
- }
73
-
74
- async function hydrateAutoShardedResult(docRef, resultName) {
75
- const shardsCol = docRef.collection('_shards');
76
- const snapshot = await shardsCol.get();
77
- const assembledData = { _completed: true };
78
- snapshot.forEach(doc => {
79
- const chunk = doc.data();
80
- // [FIX] Ensure we don't merge metadata fields that might corrupt the object
81
- const { _expireAt, ...safeChunk } = chunk;
82
- Object.assign(assembledData, safeChunk);
33
+ logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for ${dStr}`);
34
+
35
+ const results = {};
36
+ const promises = Array.from(needed).map(async (name) => {
37
+ try {
38
+ // Resolve Category from Lookup, default to 'analytics' if unknown
39
+ const category = manifestLookup[name] || 'analytics';
40
+ const data = await fetchSingleResult(db, config, dStr, name, category);
41
+ if (data) results[name] = data;
42
+ } catch (e) {
43
+ logger.log('WARN', `[DependencyFetcher] Failed to load dependency ${name}: ${e.message}`);
44
+ }
83
45
  });
84
- delete assembledData._sharded;
85
- delete assembledData._completed;
86
- return { name: resultName, data: assembledData };
46
+
47
+ await Promise.all(promises);
48
+ return results;
87
49
  }
88
50
 
89
51
  /**
90
- * [OPTIMIZED] Fetch Result Series using Batch Read
91
- * Reduces N x M reads to a single (or chunked) getAll operation.
52
+ * Fetches result series (Historical data) for lookbacks.
53
+ * @param {string} endDateStr - The most recent date.
54
+ * @param {Array} calcNames - Names of computations to fetch.
55
+ * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
92
56
  */
93
- async function fetchResultSeries(dateStr, calcsToFetchNames, fullManifest, config, deps, lookbackDays) {
94
- const { db } = deps;
95
- const results = {}; // Structure: { [date]: { [calcName]: data } }
96
- const endDate = new Date(dateStr);
57
+ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
58
+ const { db, logger } = deps;
59
+ const results = {};
60
+ const dates = [];
97
61
 
98
- // 1. Build Manifest Map for quick lookups
99
- const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
62
+ // Generate date list (starting from yesterday relative to endDateStr)
63
+ const d = new Date(endDateStr);
64
+ for (let i = 0; i < lookbackDays; i++) {
65
+ d.setUTCDate(d.getUTCDate() - 1);
66
+ dates.push(d.toISOString().slice(0, 10));
67
+ }
100
68
 
101
- // 2. Pre-calculate all Document References needed
102
- const batchRequest = [];
69
+ // Initialize structure
70
+ calcNames.forEach(name => { results[normalizeName(name)] = {}; });
103
71
 
104
- for (let i = 0; i < lookbackDays; i++) {
105
- const d = new Date(endDate);
106
- d.setUTCDate(d.getUTCDate() - i);
107
- const dString = d.toISOString().slice(0, 10);
108
-
109
- for (const name of calcsToFetchNames) {
110
- const normName = normalizeName(name);
111
- const m = manifestMap.get(normName);
112
- if (!m) continue;
72
+ logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} calcs over ${lookbackDays} days.`);
113
73
 
114
- const ref = db.collection(config.resultsCollection)
115
- .doc(dString)
116
- .collection(config.resultsSubcollection)
117
- .doc(m.category || 'unknown')
118
- .collection(config.computationsSubcollection)
119
- .doc(normName);
120
-
121
- batchRequest.push({ date: dString, name: normName, ref });
74
+ const fetchOps = [];
75
+
76
+ for (const dateStr of dates) {
77
+ for (const rawName of calcNames) {
78
+ const normName = normalizeName(rawName);
79
+ const category = manifestLookup[normName] || 'analytics';
80
+
81
+ fetchOps.push(async () => {
82
+ const val = await fetchSingleResult(db, config, dateStr, rawName, category);
83
+ if (val) {
84
+ if (!results[normName]) results[normName] = {};
85
+ results[normName][dateStr] = val;
86
+ }
87
+ });
122
88
  }
123
89
  }
90
+
91
+ // Limited concurrency batch execution (Batch size 20)
92
+ const BATCH_SIZE = 20;
93
+ for (let i = 0; i < fetchOps.length; i += BATCH_SIZE) {
94
+ await Promise.all(fetchOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
95
+ }
96
+
97
+ return results;
98
+ }
124
99
 
125
- if (batchRequest.length === 0) return {};
126
-
127
- // 3. Batch Fetch (Chunked to respect Firestore limits, usually 100-500 is safe)
128
- const BATCH_SIZE = 100;
129
- const hydrationTasks = [];
100
+ /**
101
+ * Core Helper: Fetches a single result, handles Sharding & Compression.
102
+ */
103
+ async function fetchSingleResult(db, config, dateStr, name, category) {
104
+ const docRef = db.collection(config.resultsCollection)
105
+ .doc(dateStr)
106
+ .collection(config.resultsSubcollection)
107
+ .doc(category)
108
+ .collection(config.computationsSubcollection)
109
+ .doc(name);
110
+
111
+ const snap = await docRef.get();
112
+ if (!snap.exists) return null;
130
113
 
131
- // Helper to process a batch of snapshots
132
- const processBatch = async (items) => {
133
- const refs = items.map(i => i.ref);
134
- let snapshots;
114
+ let data = snap.data();
115
+
116
+ // 1. Handle Compression
117
+ if (data._compressed && data.payload) {
135
118
  try {
136
- snapshots = await db.getAll(...refs);
119
+ const buffer = (data.payload instanceof Buffer) ? data.payload : data.payload.toDate();
120
+ const decompressed = zlib.gunzipSync(buffer);
121
+ const jsonStr = decompressed.toString('utf8');
122
+ const realData = JSON.parse(jsonStr);
123
+ // Merge decompressed data
124
+ data = { ...data, ...realData };
125
+ delete data.payload;
137
126
  } catch (e) {
138
- console.warn(`[DependencyFetcher] Batch read failed: ${e.message}. Skipping batch.`);
139
- return;
127
+ console.warn(`[DependencyFetcher] Decompression failed for ${name}: ${e.message}`);
128
+ return null;
140
129
  }
141
-
142
- for (let i = 0; i < snapshots.length; i++) {
143
- const doc = snapshots[i];
144
- const meta = items[i];
145
-
146
- if (!doc.exists) continue;
147
-
148
- const data = doc.data();
149
- let finalData = null;
150
-
151
- // A. Compressed
152
- if (data._compressed === true && data.payload) {
153
- try {
154
- const unzipped = zlib.gunzipSync(data.payload);
155
- finalData = JSON.parse(unzipped.toString());
156
- } catch (e) {
157
- console.error(`[Hydration] Failed to decompress ${meta.name} for ${meta.date}`, e);
158
- }
159
- }
160
- // B. Sharded (Defer hydration to avoid blocking the loop)
161
- else if (data._sharded === true) {
162
- hydrationTasks.push({
163
- date: meta.date,
164
- name: meta.name,
165
- ref: doc.ref
130
+ }
131
+
132
+ // 2. Handle Sharding
133
+ if (data._sharded) {
134
+ const shardCol = docRef.collection('_shards');
135
+ const shardSnaps = await shardCol.get();
136
+
137
+ if (!shardSnaps.empty) {
138
+ shardSnaps.forEach(shard => {
139
+ const shardData = shard.data();
140
+ // Merge shard contents, ignoring internal metadata if it clashes
141
+ Object.entries(shardData).forEach(([k, v]) => {
142
+ if (!k.startsWith('_')) {
143
+ data[k] = v;
144
+ }
166
145
  });
167
- continue; // Skip immediate assignment
168
- }
169
- // C. Standard
170
- else if (data._completed) {
171
- finalData = data;
172
- }
173
-
174
- // Assign if we have data
175
- if (finalData) {
176
- if (!results[meta.date]) results[meta.date] = {};
177
- results[meta.date][meta.name] = finalData;
178
- }
146
+ });
179
147
  }
180
- };
181
-
182
- // Execute batches
183
- for (let i = 0; i < batchRequest.length; i += BATCH_SIZE) {
184
- const chunk = batchRequest.slice(i, i + BATCH_SIZE);
185
- await processBatch(chunk);
186
- }
187
-
188
- // 4. Handle Sharded Results (Parallel Hydration)
189
- if (hydrationTasks.length > 0) {
190
- // Limit concurrency for shard fetching to avoid overwhelming the client
191
- const limit = pLimit(20);
192
- await Promise.all(hydrationTasks.map(task => limit(async () => {
193
- try {
194
- const res = await hydrateAutoShardedResult(task.ref, task.name);
195
- if (!results[task.date]) results[task.date] = {};
196
- results[task.date][task.name] = res.data;
197
- } catch (e) {
198
- console.warn(`[DependencyFetcher] Failed to hydrate shards for ${task.name}/${task.date}: ${e.message}`);
199
- }
200
- })));
201
148
  }
202
-
203
- return results;
149
+
150
+ return data;
204
151
  }
205
152
 
206
- module.exports = { fetchExistingResults, fetchResultSeries };
153
+ module.exports = { fetchDependencies, fetchResultSeries };