bulltrackers-module 1.0.631 → 1.0.633
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/context/ContextFactory.js +39 -18
- package/functions/computation-system/data/AvailabilityChecker.js +27 -11
- package/functions/computation-system/data/DependencyFetcher.js +125 -178
- package/functions/computation-system/executors/MetaExecutor.js +47 -98
- package/functions/computation-system/executors/StandardExecutor.js +11 -25
- package/functions/computation-system/persistence/ResultCommitter.js +4 -3
- package/functions/computation-system/utils/data_loader.js +105 -143
- package/package.json +1 -1
|
@@ -20,17 +20,32 @@ class ContextFactory {
|
|
|
20
20
|
|
|
21
21
|
static buildPerUserContext(options) {
|
|
22
22
|
const {
|
|
23
|
-
todayPortfolio,
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
todayPortfolio,
|
|
24
|
+
yesterdayPortfolio,
|
|
25
|
+
todayHistory,
|
|
26
|
+
yesterdayHistory,
|
|
27
|
+
userId,
|
|
28
|
+
userType,
|
|
29
|
+
dateStr,
|
|
30
|
+
metadata,
|
|
31
|
+
mappings,
|
|
32
|
+
insights,
|
|
33
|
+
socialData,
|
|
34
|
+
computedDependencies,
|
|
35
|
+
previousComputedDependencies,
|
|
36
|
+
config,
|
|
37
|
+
deps,
|
|
26
38
|
verification,
|
|
27
|
-
rankings,
|
|
28
|
-
|
|
39
|
+
rankings,
|
|
40
|
+
yesterdayRankings,
|
|
41
|
+
allRankings,
|
|
42
|
+
allRankingsYesterday,
|
|
29
43
|
allVerifications,
|
|
30
|
-
|
|
31
|
-
|
|
44
|
+
ratings,
|
|
45
|
+
pageViews,
|
|
46
|
+
watchlistMembership,
|
|
47
|
+
alertHistory,
|
|
32
48
|
piMasterList,
|
|
33
|
-
// [NEW] Series Data (Lookback for Root Data or Computation Results)
|
|
34
49
|
seriesData
|
|
35
50
|
} = options;
|
|
36
51
|
|
|
@@ -56,14 +71,11 @@ class ContextFactory {
|
|
|
56
71
|
rankings: allRankings || [],
|
|
57
72
|
rankingsYesterday: allRankingsYesterday || [],
|
|
58
73
|
verifications: allVerifications || {},
|
|
59
|
-
// [NEW] New Root Data Types for Profile Metrics
|
|
60
74
|
ratings: ratings || {},
|
|
61
75
|
pageViews: pageViews || {},
|
|
62
76
|
watchlistMembership: watchlistMembership || {},
|
|
63
77
|
alertHistory: alertHistory || {},
|
|
64
78
|
piMasterList: piMasterList || {},
|
|
65
|
-
// [NEW] Expose Series Data
|
|
66
|
-
// Structure: { root: { [type]: { [date]: data } }, results: { [date]: { [calcName]: data } } }
|
|
67
79
|
series: seriesData || {}
|
|
68
80
|
}
|
|
69
81
|
};
|
|
@@ -71,13 +83,23 @@ class ContextFactory {
|
|
|
71
83
|
|
|
72
84
|
static buildMetaContext(options) {
|
|
73
85
|
const {
|
|
74
|
-
dateStr,
|
|
75
|
-
|
|
76
|
-
|
|
86
|
+
dateStr,
|
|
87
|
+
metadata,
|
|
88
|
+
mappings,
|
|
89
|
+
insights,
|
|
90
|
+
socialData,
|
|
91
|
+
prices,
|
|
92
|
+
computedDependencies,
|
|
93
|
+
previousComputedDependencies,
|
|
94
|
+
config,
|
|
95
|
+
deps,
|
|
96
|
+
allRankings,
|
|
97
|
+
allRankingsYesterday,
|
|
77
98
|
allVerifications,
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
99
|
+
ratings,
|
|
100
|
+
pageViews,
|
|
101
|
+
watchlistMembership,
|
|
102
|
+
alertHistory,
|
|
81
103
|
seriesData
|
|
82
104
|
} = options;
|
|
83
105
|
|
|
@@ -99,7 +121,6 @@ class ContextFactory {
|
|
|
99
121
|
pageViews: pageViews || {},
|
|
100
122
|
watchlistMembership: watchlistMembership || {},
|
|
101
123
|
alertHistory: alertHistory || {},
|
|
102
|
-
// [NEW] Expose Series Data
|
|
103
124
|
series: seriesData || {}
|
|
104
125
|
}
|
|
105
126
|
};
|
|
@@ -30,11 +30,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
|
|
|
30
30
|
let isAvailable = false;
|
|
31
31
|
|
|
32
32
|
if (dep === 'portfolio') {
|
|
33
|
-
if (userType === 'speculator'
|
|
34
|
-
else if (userType === 'normal'
|
|
33
|
+
if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
|
|
34
|
+
else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
|
|
35
35
|
else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
|
|
36
|
-
else if (userType === 'signed_in_user'
|
|
37
|
-
else if (userType === 'all'
|
|
36
|
+
else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
|
|
37
|
+
else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
|
|
38
38
|
|
|
39
39
|
if (!isAvailable) {
|
|
40
40
|
// [OPTIMIZATION] Optimistic Series Check
|
|
@@ -284,12 +284,28 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
|
|
|
284
284
|
logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
|
|
285
285
|
return {
|
|
286
286
|
status: {
|
|
287
|
-
hasPortfolio: false,
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
287
|
+
hasPortfolio: false,
|
|
288
|
+
hasHistory: false,
|
|
289
|
+
hasSocial: false,
|
|
290
|
+
hasInsights: false,
|
|
291
|
+
hasPrices: false,
|
|
292
|
+
speculatorPortfolio: false,
|
|
293
|
+
normalPortfolio: false,
|
|
294
|
+
speculatorHistory: false,
|
|
295
|
+
normalHistory: false,
|
|
296
|
+
piRankings: false,
|
|
297
|
+
piPortfolios: false,
|
|
298
|
+
piDeepPortfolios: false,
|
|
299
|
+
piHistory: false,
|
|
300
|
+
signedInUserPortfolio: false,
|
|
301
|
+
signedInUserHistory: false,
|
|
302
|
+
signedInUserVerification: false,
|
|
303
|
+
hasPISocial: false,
|
|
304
|
+
hasSignedInSocial: false,
|
|
305
|
+
piRatings: false,
|
|
306
|
+
piPageViews: false,
|
|
307
|
+
watchlistMembership: false,
|
|
308
|
+
piAlertHistory: false
|
|
293
309
|
}
|
|
294
310
|
};
|
|
295
311
|
}
|
|
@@ -361,5 +377,5 @@ module.exports = {
|
|
|
361
377
|
checkRootDependencies,
|
|
362
378
|
checkRootDataAvailability,
|
|
363
379
|
getViableCalculations,
|
|
364
|
-
getAvailabilityWindow
|
|
380
|
+
getAvailabilityWindow
|
|
365
381
|
};
|
|
@@ -1,206 +1,153 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @fileoverview Fetches
|
|
3
|
-
* UPDATED:
|
|
2
|
+
* @fileoverview Fetches dependencies for computations.
|
|
3
|
+
* UPDATED: Uses 'manifestLookup' to resolve the correct category (Core vs Non-Core).
|
|
4
|
+
* UPDATED: Supports automatic reassembly of sharded results (_shards subcollection).
|
|
5
|
+
* UPDATED: Supports decompression of zipped results.
|
|
4
6
|
*/
|
|
5
7
|
const { normalizeName } = require('../utils/utils');
|
|
6
|
-
const zlib = require('zlib');
|
|
7
|
-
const pLimit = require('p-limit');
|
|
8
|
+
const zlib = require('zlib');
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Fetches dependencies for a specific date (Standard pass).
|
|
12
|
+
* @param {Date} date - The target date.
|
|
13
|
+
* @param {Array} calcs - The computations requiring dependencies.
|
|
14
|
+
* @param {Object} config - System config.
|
|
15
|
+
* @param {Object} deps - System dependencies (db, logger).
|
|
16
|
+
* @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
|
|
17
|
+
*/
|
|
18
|
+
async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}) {
|
|
19
|
+
const { db, logger } = deps;
|
|
20
|
+
const dStr = date.toISOString().slice(0, 10);
|
|
12
21
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
22
|
+
// 1. Identify unique dependencies needed
|
|
23
|
+
const needed = new Set();
|
|
24
|
+
calcs.forEach(c => {
|
|
25
|
+
if (c.getDependencies) {
|
|
26
|
+
const reqs = c.getDependencies();
|
|
27
|
+
reqs.forEach(r => needed.add(normalizeName(r)));
|
|
28
|
+
}
|
|
29
|
+
});
|
|
17
30
|
|
|
18
|
-
if (
|
|
31
|
+
if (needed.size === 0) return {};
|
|
19
32
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
.doc(name));
|
|
33
|
-
names.push(name);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
if (docRefs.length) {
|
|
38
|
-
const snaps = await db.getAll(...docRefs);
|
|
39
|
-
const hydrationPromises = [];
|
|
40
|
-
|
|
41
|
-
snaps.forEach((doc, i) => {
|
|
42
|
-
const name = names[i];
|
|
43
|
-
if (!doc.exists) return;
|
|
44
|
-
const data = doc.data();
|
|
45
|
-
|
|
46
|
-
// Handle Decompression
|
|
47
|
-
if (data._compressed === true && data.payload) {
|
|
48
|
-
try {
|
|
49
|
-
const unzipped = zlib.gunzipSync(data.payload);
|
|
50
|
-
fetched[name] = JSON.parse(unzipped.toString());
|
|
51
|
-
} catch (e) {
|
|
52
|
-
console.error(`[Hydration] Failed to decompress ${name}:`, e);
|
|
53
|
-
fetched[name] = {};
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
// Handle Sharding
|
|
57
|
-
else if (data._sharded === true) {
|
|
58
|
-
hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name));
|
|
59
|
-
}
|
|
60
|
-
// Standard
|
|
61
|
-
else if (data._completed) {
|
|
62
|
-
fetched[name] = data;
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
if (hydrationPromises.length > 0) {
|
|
67
|
-
const hydratedResults = await Promise.all(hydrationPromises);
|
|
68
|
-
hydratedResults.forEach(res => { fetched[res.name] = res.data; });
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
return fetched;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async function hydrateAutoShardedResult(docRef, resultName) {
|
|
75
|
-
const shardsCol = docRef.collection('_shards');
|
|
76
|
-
const snapshot = await shardsCol.get();
|
|
77
|
-
const assembledData = { _completed: true };
|
|
78
|
-
snapshot.forEach(doc => {
|
|
79
|
-
const chunk = doc.data();
|
|
80
|
-
// [FIX] Ensure we don't merge metadata fields that might corrupt the object
|
|
81
|
-
const { _expireAt, ...safeChunk } = chunk;
|
|
82
|
-
Object.assign(assembledData, safeChunk);
|
|
33
|
+
logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for ${dStr}`);
|
|
34
|
+
|
|
35
|
+
const results = {};
|
|
36
|
+
const promises = Array.from(needed).map(async (name) => {
|
|
37
|
+
try {
|
|
38
|
+
// Resolve Category from Lookup, default to 'analytics' if unknown
|
|
39
|
+
const category = manifestLookup[name] || 'analytics';
|
|
40
|
+
const data = await fetchSingleResult(db, config, dStr, name, category);
|
|
41
|
+
if (data) results[name] = data;
|
|
42
|
+
} catch (e) {
|
|
43
|
+
logger.log('WARN', `[DependencyFetcher] Failed to load dependency ${name}: ${e.message}`);
|
|
44
|
+
}
|
|
83
45
|
});
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
return
|
|
46
|
+
|
|
47
|
+
await Promise.all(promises);
|
|
48
|
+
return results;
|
|
87
49
|
}
|
|
88
50
|
|
|
89
51
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
52
|
+
* Fetches result series (Historical data) for lookbacks.
|
|
53
|
+
* @param {string} endDateStr - The most recent date.
|
|
54
|
+
* @param {Array} calcNames - Names of computations to fetch.
|
|
55
|
+
* @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
|
|
92
56
|
*/
|
|
93
|
-
async function fetchResultSeries(
|
|
94
|
-
const { db } = deps;
|
|
95
|
-
const results = {};
|
|
96
|
-
const
|
|
57
|
+
async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
|
|
58
|
+
const { db, logger } = deps;
|
|
59
|
+
const results = {};
|
|
60
|
+
const dates = [];
|
|
97
61
|
|
|
98
|
-
//
|
|
99
|
-
const
|
|
62
|
+
// Generate date list (starting from yesterday relative to endDateStr)
|
|
63
|
+
const d = new Date(endDateStr);
|
|
64
|
+
for (let i = 0; i < lookbackDays; i++) {
|
|
65
|
+
d.setUTCDate(d.getUTCDate() - 1);
|
|
66
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
67
|
+
}
|
|
100
68
|
|
|
101
|
-
//
|
|
102
|
-
|
|
69
|
+
// Initialize structure
|
|
70
|
+
calcNames.forEach(name => { results[normalizeName(name)] = {}; });
|
|
103
71
|
|
|
104
|
-
|
|
105
|
-
const d = new Date(endDate);
|
|
106
|
-
d.setUTCDate(d.getUTCDate() - i);
|
|
107
|
-
const dString = d.toISOString().slice(0, 10);
|
|
108
|
-
|
|
109
|
-
for (const name of calcsToFetchNames) {
|
|
110
|
-
const normName = normalizeName(name);
|
|
111
|
-
const m = manifestMap.get(normName);
|
|
112
|
-
if (!m) continue;
|
|
72
|
+
logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} calcs over ${lookbackDays} days.`);
|
|
113
73
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
74
|
+
const fetchOps = [];
|
|
75
|
+
|
|
76
|
+
for (const dateStr of dates) {
|
|
77
|
+
for (const rawName of calcNames) {
|
|
78
|
+
const normName = normalizeName(rawName);
|
|
79
|
+
const category = manifestLookup[normName] || 'analytics';
|
|
80
|
+
|
|
81
|
+
fetchOps.push(async () => {
|
|
82
|
+
const val = await fetchSingleResult(db, config, dateStr, rawName, category);
|
|
83
|
+
if (val) {
|
|
84
|
+
if (!results[normName]) results[normName] = {};
|
|
85
|
+
results[normName][dateStr] = val;
|
|
86
|
+
}
|
|
87
|
+
});
|
|
122
88
|
}
|
|
123
89
|
}
|
|
90
|
+
|
|
91
|
+
// Limited concurrency batch execution (Batch size 20)
|
|
92
|
+
const BATCH_SIZE = 20;
|
|
93
|
+
for (let i = 0; i < fetchOps.length; i += BATCH_SIZE) {
|
|
94
|
+
await Promise.all(fetchOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return results;
|
|
98
|
+
}
|
|
124
99
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const
|
|
100
|
+
/**
|
|
101
|
+
* Core Helper: Fetches a single result, handles Sharding & Compression.
|
|
102
|
+
*/
|
|
103
|
+
async function fetchSingleResult(db, config, dateStr, name, category) {
|
|
104
|
+
const docRef = db.collection(config.resultsCollection)
|
|
105
|
+
.doc(dateStr)
|
|
106
|
+
.collection(config.resultsSubcollection)
|
|
107
|
+
.doc(category)
|
|
108
|
+
.collection(config.computationsSubcollection)
|
|
109
|
+
.doc(name);
|
|
110
|
+
|
|
111
|
+
const snap = await docRef.get();
|
|
112
|
+
if (!snap.exists) return null;
|
|
130
113
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
114
|
+
let data = snap.data();
|
|
115
|
+
|
|
116
|
+
// 1. Handle Compression
|
|
117
|
+
if (data._compressed && data.payload) {
|
|
135
118
|
try {
|
|
136
|
-
|
|
119
|
+
const buffer = (data.payload instanceof Buffer) ? data.payload : data.payload.toDate();
|
|
120
|
+
const decompressed = zlib.gunzipSync(buffer);
|
|
121
|
+
const jsonStr = decompressed.toString('utf8');
|
|
122
|
+
const realData = JSON.parse(jsonStr);
|
|
123
|
+
// Merge decompressed data
|
|
124
|
+
data = { ...data, ...realData };
|
|
125
|
+
delete data.payload;
|
|
137
126
|
} catch (e) {
|
|
138
|
-
console.warn(`[DependencyFetcher]
|
|
139
|
-
return;
|
|
127
|
+
console.warn(`[DependencyFetcher] Decompression failed for ${name}: ${e.message}`);
|
|
128
|
+
return null;
|
|
140
129
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
} catch (e) {
|
|
157
|
-
console.error(`[Hydration] Failed to decompress ${meta.name} for ${meta.date}`, e);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
// B. Sharded (Defer hydration to avoid blocking the loop)
|
|
161
|
-
else if (data._sharded === true) {
|
|
162
|
-
hydrationTasks.push({
|
|
163
|
-
date: meta.date,
|
|
164
|
-
name: meta.name,
|
|
165
|
-
ref: doc.ref
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// 2. Handle Sharding
|
|
133
|
+
if (data._sharded) {
|
|
134
|
+
const shardCol = docRef.collection('_shards');
|
|
135
|
+
const shardSnaps = await shardCol.get();
|
|
136
|
+
|
|
137
|
+
if (!shardSnaps.empty) {
|
|
138
|
+
shardSnaps.forEach(shard => {
|
|
139
|
+
const shardData = shard.data();
|
|
140
|
+
// Merge shard contents, ignoring internal metadata if it clashes
|
|
141
|
+
Object.entries(shardData).forEach(([k, v]) => {
|
|
142
|
+
if (!k.startsWith('_')) {
|
|
143
|
+
data[k] = v;
|
|
144
|
+
}
|
|
166
145
|
});
|
|
167
|
-
|
|
168
|
-
}
|
|
169
|
-
// C. Standard
|
|
170
|
-
else if (data._completed) {
|
|
171
|
-
finalData = data;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
// Assign if we have data
|
|
175
|
-
if (finalData) {
|
|
176
|
-
if (!results[meta.date]) results[meta.date] = {};
|
|
177
|
-
results[meta.date][meta.name] = finalData;
|
|
178
|
-
}
|
|
146
|
+
});
|
|
179
147
|
}
|
|
180
|
-
};
|
|
181
|
-
|
|
182
|
-
// Execute batches
|
|
183
|
-
for (let i = 0; i < batchRequest.length; i += BATCH_SIZE) {
|
|
184
|
-
const chunk = batchRequest.slice(i, i + BATCH_SIZE);
|
|
185
|
-
await processBatch(chunk);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
// 4. Handle Sharded Results (Parallel Hydration)
|
|
189
|
-
if (hydrationTasks.length > 0) {
|
|
190
|
-
// Limit concurrency for shard fetching to avoid overwhelming the client
|
|
191
|
-
const limit = pLimit(20);
|
|
192
|
-
await Promise.all(hydrationTasks.map(task => limit(async () => {
|
|
193
|
-
try {
|
|
194
|
-
const res = await hydrateAutoShardedResult(task.ref, task.name);
|
|
195
|
-
if (!results[task.date]) results[task.date] = {};
|
|
196
|
-
results[task.date][task.name] = res.data;
|
|
197
|
-
} catch (e) {
|
|
198
|
-
console.warn(`[DependencyFetcher] Failed to hydrate shards for ${task.name}/${task.date}: ${e.message}`);
|
|
199
|
-
}
|
|
200
|
-
})));
|
|
201
148
|
}
|
|
202
|
-
|
|
203
|
-
return
|
|
149
|
+
|
|
150
|
+
return data;
|
|
204
151
|
}
|
|
205
152
|
|
|
206
|
-
module.exports = {
|
|
153
|
+
module.exports = { fetchDependencies, fetchResultSeries };
|