bulltrackers-module 1.0.632 → 1.0.634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/functions/computation-system/data/DependencyFetcher.js +125 -178
- package/functions/computation-system/executors/MetaExecutor.js +34 -96
- package/functions/computation-system/executors/StandardExecutor.js +8 -16
- package/functions/computation-system/system_epoch.js +1 -1
- package/package.json +1 -1
|
@@ -1,206 +1,153 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @fileoverview Fetches
|
|
3
|
-
* UPDATED:
|
|
2
|
+
* @fileoverview Fetches dependencies for computations.
|
|
3
|
+
* UPDATED: Uses 'manifestLookup' to resolve the correct category (Core vs Non-Core).
|
|
4
|
+
* UPDATED: Supports automatic reassembly of sharded results (_shards subcollection).
|
|
5
|
+
* UPDATED: Supports decompression of zipped results.
|
|
4
6
|
*/
|
|
5
7
|
const { normalizeName } = require('../utils/utils');
|
|
6
|
-
const zlib = require('zlib');
|
|
7
|
-
const pLimit = require('p-limit');
|
|
8
|
+
const zlib = require('zlib');
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Fetches dependencies for a specific date (Standard pass).
|
|
12
|
+
* @param {Date} date - The target date.
|
|
13
|
+
* @param {Array} calcs - The computations requiring dependencies.
|
|
14
|
+
* @param {Object} config - System config.
|
|
15
|
+
* @param {Object} deps - System dependencies (db, logger).
|
|
16
|
+
* @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
|
|
17
|
+
*/
|
|
18
|
+
async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}) {
|
|
19
|
+
const { db, logger } = deps;
|
|
20
|
+
const dStr = date.toISOString().slice(0, 10);
|
|
12
21
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
22
|
+
// 1. Identify unique dependencies needed
|
|
23
|
+
const needed = new Set();
|
|
24
|
+
calcs.forEach(c => {
|
|
25
|
+
if (c.getDependencies) {
|
|
26
|
+
const reqs = c.getDependencies();
|
|
27
|
+
reqs.forEach(r => needed.add(normalizeName(r)));
|
|
28
|
+
}
|
|
29
|
+
});
|
|
17
30
|
|
|
18
|
-
if (
|
|
31
|
+
if (needed.size === 0) return {};
|
|
19
32
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
.doc(name));
|
|
33
|
-
names.push(name);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
if (docRefs.length) {
|
|
38
|
-
const snaps = await db.getAll(...docRefs);
|
|
39
|
-
const hydrationPromises = [];
|
|
40
|
-
|
|
41
|
-
snaps.forEach((doc, i) => {
|
|
42
|
-
const name = names[i];
|
|
43
|
-
if (!doc.exists) return;
|
|
44
|
-
const data = doc.data();
|
|
45
|
-
|
|
46
|
-
// Handle Decompression
|
|
47
|
-
if (data._compressed === true && data.payload) {
|
|
48
|
-
try {
|
|
49
|
-
const unzipped = zlib.gunzipSync(data.payload);
|
|
50
|
-
fetched[name] = JSON.parse(unzipped.toString());
|
|
51
|
-
} catch (e) {
|
|
52
|
-
console.error(`[Hydration] Failed to decompress ${name}:`, e);
|
|
53
|
-
fetched[name] = {};
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
// Handle Sharding
|
|
57
|
-
else if (data._sharded === true) {
|
|
58
|
-
hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name));
|
|
59
|
-
}
|
|
60
|
-
// Standard
|
|
61
|
-
else if (data._completed) {
|
|
62
|
-
fetched[name] = data;
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
if (hydrationPromises.length > 0) {
|
|
67
|
-
const hydratedResults = await Promise.all(hydrationPromises);
|
|
68
|
-
hydratedResults.forEach(res => { fetched[res.name] = res.data; });
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
return fetched;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
async function hydrateAutoShardedResult(docRef, resultName) {
|
|
75
|
-
const shardsCol = docRef.collection('_shards');
|
|
76
|
-
const snapshot = await shardsCol.get();
|
|
77
|
-
const assembledData = { _completed: true };
|
|
78
|
-
snapshot.forEach(doc => {
|
|
79
|
-
const chunk = doc.data();
|
|
80
|
-
// [FIX] Ensure we don't merge metadata fields that might corrupt the object
|
|
81
|
-
const { _expireAt, ...safeChunk } = chunk;
|
|
82
|
-
Object.assign(assembledData, safeChunk);
|
|
33
|
+
logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for ${dStr}`);
|
|
34
|
+
|
|
35
|
+
const results = {};
|
|
36
|
+
const promises = Array.from(needed).map(async (name) => {
|
|
37
|
+
try {
|
|
38
|
+
// Resolve Category from Lookup, default to 'analytics' if unknown
|
|
39
|
+
const category = manifestLookup[name] || 'analytics';
|
|
40
|
+
const data = await fetchSingleResult(db, config, dStr, name, category);
|
|
41
|
+
if (data) results[name] = data;
|
|
42
|
+
} catch (e) {
|
|
43
|
+
logger.log('WARN', `[DependencyFetcher] Failed to load dependency ${name}: ${e.message}`);
|
|
44
|
+
}
|
|
83
45
|
});
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
return
|
|
46
|
+
|
|
47
|
+
await Promise.all(promises);
|
|
48
|
+
return results;
|
|
87
49
|
}
|
|
88
50
|
|
|
89
51
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
52
|
+
* Fetches result series (Historical data) for lookbacks.
|
|
53
|
+
* @param {string} endDateStr - The most recent date.
|
|
54
|
+
* @param {Array} calcNames - Names of computations to fetch.
|
|
55
|
+
* @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
|
|
92
56
|
*/
|
|
93
|
-
async function fetchResultSeries(
|
|
94
|
-
const { db } = deps;
|
|
95
|
-
const results = {};
|
|
96
|
-
const
|
|
57
|
+
async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
|
|
58
|
+
const { db, logger } = deps;
|
|
59
|
+
const results = {};
|
|
60
|
+
const dates = [];
|
|
97
61
|
|
|
98
|
-
//
|
|
99
|
-
const
|
|
62
|
+
// Generate date list (starting from yesterday relative to endDateStr)
|
|
63
|
+
const d = new Date(endDateStr);
|
|
64
|
+
for (let i = 0; i < lookbackDays; i++) {
|
|
65
|
+
d.setUTCDate(d.getUTCDate() - 1);
|
|
66
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
67
|
+
}
|
|
100
68
|
|
|
101
|
-
//
|
|
102
|
-
|
|
69
|
+
// Initialize structure
|
|
70
|
+
calcNames.forEach(name => { results[normalizeName(name)] = {}; });
|
|
103
71
|
|
|
104
|
-
|
|
105
|
-
const d = new Date(endDate);
|
|
106
|
-
d.setUTCDate(d.getUTCDate() - i);
|
|
107
|
-
const dString = d.toISOString().slice(0, 10);
|
|
108
|
-
|
|
109
|
-
for (const name of calcsToFetchNames) {
|
|
110
|
-
const normName = normalizeName(name);
|
|
111
|
-
const m = manifestMap.get(normName);
|
|
112
|
-
if (!m) continue;
|
|
72
|
+
logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} calcs over ${lookbackDays} days.`);
|
|
113
73
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
74
|
+
const fetchOps = [];
|
|
75
|
+
|
|
76
|
+
for (const dateStr of dates) {
|
|
77
|
+
for (const rawName of calcNames) {
|
|
78
|
+
const normName = normalizeName(rawName);
|
|
79
|
+
const category = manifestLookup[normName] || 'analytics';
|
|
80
|
+
|
|
81
|
+
fetchOps.push(async () => {
|
|
82
|
+
const val = await fetchSingleResult(db, config, dateStr, rawName, category);
|
|
83
|
+
if (val) {
|
|
84
|
+
if (!results[normName]) results[normName] = {};
|
|
85
|
+
results[normName][dateStr] = val;
|
|
86
|
+
}
|
|
87
|
+
});
|
|
122
88
|
}
|
|
123
89
|
}
|
|
90
|
+
|
|
91
|
+
// Limited concurrency batch execution (Batch size 20)
|
|
92
|
+
const BATCH_SIZE = 20;
|
|
93
|
+
for (let i = 0; i < fetchOps.length; i += BATCH_SIZE) {
|
|
94
|
+
await Promise.all(fetchOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return results;
|
|
98
|
+
}
|
|
124
99
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const
|
|
100
|
+
/**
|
|
101
|
+
* Core Helper: Fetches a single result, handles Sharding & Compression.
|
|
102
|
+
*/
|
|
103
|
+
async function fetchSingleResult(db, config, dateStr, name, category) {
|
|
104
|
+
const docRef = db.collection(config.resultsCollection)
|
|
105
|
+
.doc(dateStr)
|
|
106
|
+
.collection(config.resultsSubcollection)
|
|
107
|
+
.doc(category)
|
|
108
|
+
.collection(config.computationsSubcollection)
|
|
109
|
+
.doc(name);
|
|
110
|
+
|
|
111
|
+
const snap = await docRef.get();
|
|
112
|
+
if (!snap.exists) return null;
|
|
130
113
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
114
|
+
let data = snap.data();
|
|
115
|
+
|
|
116
|
+
// 1. Handle Compression
|
|
117
|
+
if (data._compressed && data.payload) {
|
|
135
118
|
try {
|
|
136
|
-
|
|
119
|
+
const buffer = (data.payload instanceof Buffer) ? data.payload : data.payload.toDate();
|
|
120
|
+
const decompressed = zlib.gunzipSync(buffer);
|
|
121
|
+
const jsonStr = decompressed.toString('utf8');
|
|
122
|
+
const realData = JSON.parse(jsonStr);
|
|
123
|
+
// Merge decompressed data
|
|
124
|
+
data = { ...data, ...realData };
|
|
125
|
+
delete data.payload;
|
|
137
126
|
} catch (e) {
|
|
138
|
-
console.warn(`[DependencyFetcher]
|
|
139
|
-
return;
|
|
127
|
+
console.warn(`[DependencyFetcher] Decompression failed for ${name}: ${e.message}`);
|
|
128
|
+
return null;
|
|
140
129
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
} catch (e) {
|
|
157
|
-
console.error(`[Hydration] Failed to decompress ${meta.name} for ${meta.date}`, e);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
// B. Sharded (Defer hydration to avoid blocking the loop)
|
|
161
|
-
else if (data._sharded === true) {
|
|
162
|
-
hydrationTasks.push({
|
|
163
|
-
date: meta.date,
|
|
164
|
-
name: meta.name,
|
|
165
|
-
ref: doc.ref
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// 2. Handle Sharding
|
|
133
|
+
if (data._sharded) {
|
|
134
|
+
const shardCol = docRef.collection('_shards');
|
|
135
|
+
const shardSnaps = await shardCol.get();
|
|
136
|
+
|
|
137
|
+
if (!shardSnaps.empty) {
|
|
138
|
+
shardSnaps.forEach(shard => {
|
|
139
|
+
const shardData = shard.data();
|
|
140
|
+
// Merge shard contents, ignoring internal metadata if it clashes
|
|
141
|
+
Object.entries(shardData).forEach(([k, v]) => {
|
|
142
|
+
if (!k.startsWith('_')) {
|
|
143
|
+
data[k] = v;
|
|
144
|
+
}
|
|
166
145
|
});
|
|
167
|
-
|
|
168
|
-
}
|
|
169
|
-
// C. Standard
|
|
170
|
-
else if (data._completed) {
|
|
171
|
-
finalData = data;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
// Assign if we have data
|
|
175
|
-
if (finalData) {
|
|
176
|
-
if (!results[meta.date]) results[meta.date] = {};
|
|
177
|
-
results[meta.date][meta.name] = finalData;
|
|
178
|
-
}
|
|
146
|
+
});
|
|
179
147
|
}
|
|
180
|
-
};
|
|
181
|
-
|
|
182
|
-
// Execute batches
|
|
183
|
-
for (let i = 0; i < batchRequest.length; i += BATCH_SIZE) {
|
|
184
|
-
const chunk = batchRequest.slice(i, i + BATCH_SIZE);
|
|
185
|
-
await processBatch(chunk);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
// 4. Handle Sharded Results (Parallel Hydration)
|
|
189
|
-
if (hydrationTasks.length > 0) {
|
|
190
|
-
// Limit concurrency for shard fetching to avoid overwhelming the client
|
|
191
|
-
const limit = pLimit(20);
|
|
192
|
-
await Promise.all(hydrationTasks.map(task => limit(async () => {
|
|
193
|
-
try {
|
|
194
|
-
const res = await hydrateAutoShardedResult(task.ref, task.name);
|
|
195
|
-
if (!results[task.date]) results[task.date] = {};
|
|
196
|
-
results[task.date][task.name] = res.data;
|
|
197
|
-
} catch (e) {
|
|
198
|
-
console.warn(`[DependencyFetcher] Failed to hydrate shards for ${task.name}/${task.date}: ${e.message}`);
|
|
199
|
-
}
|
|
200
|
-
})));
|
|
201
148
|
}
|
|
202
|
-
|
|
203
|
-
return
|
|
149
|
+
|
|
150
|
+
return data;
|
|
204
151
|
}
|
|
205
152
|
|
|
206
|
-
module.exports = {
|
|
153
|
+
module.exports = { fetchDependencies, fetchResultSeries };
|
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
* UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
|
|
6
6
|
* UPDATED: Support for historical rankings in Meta Context.
|
|
7
7
|
* UPDATED: Added support for loading Series Data (Root & Results) for lookbacks.
|
|
8
|
+
* UPDATED: Builds Manifest Lookup for DependencyFetcher.
|
|
8
9
|
*/
|
|
9
10
|
const { normalizeName } = require('../utils/utils');
|
|
10
11
|
const { CachedDataLoader } = require('../data/CachedDataLoader');
|
|
11
12
|
const { ContextFactory } = require('../context/ContextFactory');
|
|
12
13
|
const { commitResults } = require('../persistence/ResultCommitter');
|
|
13
|
-
const { fetchResultSeries } = require('../data/DependencyFetcher');
|
|
14
|
+
const { fetchResultSeries, fetchDependencies } = require('../data/DependencyFetcher');
|
|
15
|
+
const { getManifest } = require('../topology/ManifestLoader');
|
|
14
16
|
|
|
15
17
|
class MetaExecutor {
|
|
16
18
|
static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
|
|
@@ -18,6 +20,12 @@ class MetaExecutor {
|
|
|
18
20
|
const { logger, db } = deps;
|
|
19
21
|
const loader = new CachedDataLoader(config, deps);
|
|
20
22
|
|
|
23
|
+
// --- [FIXED] Build Global Manifest Lookup using getManifest ---
|
|
24
|
+
// getManifest is typically cached, so this is cheap.
|
|
25
|
+
const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps);
|
|
26
|
+
const manifestLookup = {};
|
|
27
|
+
allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
|
|
28
|
+
|
|
21
29
|
// [FIX] Check if any meta calculation needs history
|
|
22
30
|
const needsHistory = calcs.some(c => c.isHistorical);
|
|
23
31
|
let rankingsYesterday = null;
|
|
@@ -51,16 +59,11 @@ class MetaExecutor {
|
|
|
51
59
|
// Ratings
|
|
52
60
|
if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
|
|
53
61
|
loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
|
|
54
|
-
// Only catch if ALL calcs allow missing roots
|
|
55
62
|
const allAllowMissing = calcs.every(c => {
|
|
56
63
|
const needsRatings = c.rootDataDependencies?.includes('ratings');
|
|
57
64
|
return !needsRatings || c.canHaveMissingRoots === true;
|
|
58
65
|
});
|
|
59
|
-
if (allAllowMissing) {
|
|
60
|
-
ratings = null;
|
|
61
|
-
} else {
|
|
62
|
-
throw e; // Re-throw if any calc requires this data
|
|
63
|
-
}
|
|
66
|
+
if (allAllowMissing) { ratings = null; } else { throw e; }
|
|
64
67
|
}));
|
|
65
68
|
}
|
|
66
69
|
|
|
@@ -71,11 +74,7 @@ class MetaExecutor {
|
|
|
71
74
|
const needsPageViews = c.rootDataDependencies?.includes('pageViews');
|
|
72
75
|
return !needsPageViews || c.canHaveMissingRoots === true;
|
|
73
76
|
});
|
|
74
|
-
if (allAllowMissing) {
|
|
75
|
-
pageViews = null;
|
|
76
|
-
} else {
|
|
77
|
-
throw e;
|
|
78
|
-
}
|
|
77
|
+
if (allAllowMissing) { pageViews = null; } else { throw e; }
|
|
79
78
|
}));
|
|
80
79
|
}
|
|
81
80
|
|
|
@@ -86,11 +85,7 @@ class MetaExecutor {
|
|
|
86
85
|
const needsWatchlist = c.rootDataDependencies?.includes('watchlist');
|
|
87
86
|
return !needsWatchlist || c.canHaveMissingRoots === true;
|
|
88
87
|
});
|
|
89
|
-
if (allAllowMissing) {
|
|
90
|
-
watchlistMembership = null;
|
|
91
|
-
} else {
|
|
92
|
-
throw e;
|
|
93
|
-
}
|
|
88
|
+
if (allAllowMissing) { watchlistMembership = null; } else { throw e; }
|
|
94
89
|
}));
|
|
95
90
|
}
|
|
96
91
|
|
|
@@ -101,11 +96,7 @@ class MetaExecutor {
|
|
|
101
96
|
const needsAlerts = c.rootDataDependencies?.includes('alerts');
|
|
102
97
|
return !needsAlerts || c.canHaveMissingRoots === true;
|
|
103
98
|
});
|
|
104
|
-
if (allAllowMissing) {
|
|
105
|
-
alertHistory = null;
|
|
106
|
-
} else {
|
|
107
|
-
throw e;
|
|
108
|
-
}
|
|
99
|
+
if (allAllowMissing) { alertHistory = null; } else { throw e; }
|
|
109
100
|
}));
|
|
110
101
|
}
|
|
111
102
|
|
|
@@ -115,23 +106,13 @@ class MetaExecutor {
|
|
|
115
106
|
for (const c of calcs) {
|
|
116
107
|
const deps = c.rootDataDependencies || [];
|
|
117
108
|
const canSkip = c.canHaveMissingRoots === true;
|
|
118
|
-
|
|
119
|
-
// Helper to check if a specific root is missing
|
|
120
109
|
const isMissing = (key, val) => deps.includes(key) && (val === null || val === undefined);
|
|
121
110
|
|
|
122
111
|
if (!canSkip) {
|
|
123
|
-
if (isMissing('ratings', ratings)) {
|
|
124
|
-
|
|
125
|
-
}
|
|
126
|
-
if (isMissing('
|
|
127
|
-
throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
|
|
128
|
-
}
|
|
129
|
-
if (isMissing('watchlist', watchlistMembership)) {
|
|
130
|
-
throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
|
|
131
|
-
}
|
|
132
|
-
if (isMissing('alerts', alertHistory)) {
|
|
133
|
-
throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
|
|
134
|
-
}
|
|
112
|
+
if (isMissing('ratings', ratings)) throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
|
|
113
|
+
if (isMissing('pageViews', pageViews)) throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
|
|
114
|
+
if (isMissing('watchlist', watchlistMembership)) throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
|
|
115
|
+
if (isMissing('alerts', alertHistory)) throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
|
|
135
116
|
}
|
|
136
117
|
}
|
|
137
118
|
}
|
|
@@ -142,7 +123,6 @@ class MetaExecutor {
|
|
|
142
123
|
|
|
143
124
|
// 1. Identify requirements from manifests
|
|
144
125
|
calcs.forEach(c => {
|
|
145
|
-
// Check for Root Data Series
|
|
146
126
|
if (c.rootDataSeries) {
|
|
147
127
|
Object.entries(c.rootDataSeries).forEach(([type, conf]) => {
|
|
148
128
|
const days = typeof conf === 'object' ? conf.lookback : conf;
|
|
@@ -151,7 +131,6 @@ class MetaExecutor {
|
|
|
151
131
|
}
|
|
152
132
|
});
|
|
153
133
|
}
|
|
154
|
-
// Check for Computation Result Series
|
|
155
134
|
if (c.dependencySeries) {
|
|
156
135
|
Object.entries(c.dependencySeries).forEach(([depName, conf]) => {
|
|
157
136
|
const days = typeof conf === 'object' ? conf.lookback : conf;
|
|
@@ -177,9 +156,8 @@ class MetaExecutor {
|
|
|
177
156
|
|
|
178
157
|
if (loaderMethod) {
|
|
179
158
|
logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
|
|
180
|
-
// Assume CachedDataLoader has loadSeries method (added in previous step)
|
|
181
159
|
const series = await loader.loadSeries(loaderMethod, dStr, days);
|
|
182
|
-
seriesData.root[type] = series.data;
|
|
160
|
+
seriesData.root[type] = series.data;
|
|
183
161
|
}
|
|
184
162
|
}
|
|
185
163
|
|
|
@@ -189,11 +167,10 @@ class MetaExecutor {
|
|
|
189
167
|
const maxDays = Math.max(...Object.values(dependencySeriesRequests));
|
|
190
168
|
logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
|
|
191
169
|
|
|
192
|
-
//
|
|
193
|
-
const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch,
|
|
194
|
-
seriesData.results = resultsSeries;
|
|
170
|
+
// Pass manifestLookup to fetcher
|
|
171
|
+
const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
|
|
172
|
+
seriesData.results = resultsSeries;
|
|
195
173
|
}
|
|
196
|
-
// ---------------------------------------------
|
|
197
174
|
|
|
198
175
|
const state = {};
|
|
199
176
|
for (const c of calcs) {
|
|
@@ -210,21 +187,17 @@ class MetaExecutor {
|
|
|
210
187
|
previousComputedDependencies: previousFetchedDeps,
|
|
211
188
|
config, deps,
|
|
212
189
|
allRankings: rankings,
|
|
213
|
-
allRankingsYesterday: rankingsYesterday,
|
|
190
|
+
allRankingsYesterday: rankingsYesterday,
|
|
214
191
|
allVerifications: verifications,
|
|
215
|
-
// [NEW] Pass New Root Data Types for Profile Metrics
|
|
216
192
|
ratings: ratings || {},
|
|
217
193
|
pageViews: pageViews || {},
|
|
218
194
|
watchlistMembership: watchlistMembership || {},
|
|
219
195
|
alertHistory: alertHistory || {},
|
|
220
|
-
// [NEW] Pass Series Data
|
|
221
196
|
seriesData
|
|
222
197
|
});
|
|
223
198
|
|
|
224
199
|
try {
|
|
225
200
|
const result = await inst.process(context);
|
|
226
|
-
// Meta results are usually wrapped in a global key or just the result object
|
|
227
|
-
// The structure below implies we store it under the date key
|
|
228
201
|
inst.results = { [dStr]: { global: result } };
|
|
229
202
|
state[c.name] = inst;
|
|
230
203
|
} catch (e) {
|
|
@@ -244,7 +217,6 @@ class MetaExecutor {
|
|
|
244
217
|
const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
|
|
245
218
|
const social = metadata.rootDataDependencies?.includes('social') ? { today: await loader.loadSocial(dateStr) } : null;
|
|
246
219
|
|
|
247
|
-
// [FIX] Historical support for Batch/OncePerDay execution
|
|
248
220
|
let rankingsYesterday = null;
|
|
249
221
|
if (metadata.isHistorical) {
|
|
250
222
|
const prevDate = new Date(dateStr);
|
|
@@ -253,67 +225,37 @@ class MetaExecutor {
|
|
|
253
225
|
rankingsYesterday = await loader.loadRankings(prevStr);
|
|
254
226
|
}
|
|
255
227
|
|
|
256
|
-
// Load current rankings (often needed for ContextFactory.buildMetaContext)
|
|
257
228
|
const rankings = await loader.loadRankings(dateStr);
|
|
258
|
-
|
|
259
|
-
// [NEW] Load New Root Data Types for Profile Metrics
|
|
260
|
-
// [FIX] Enforce canHaveMissingRoots
|
|
261
229
|
const allowMissing = metadata.canHaveMissingRoots === true;
|
|
262
230
|
|
|
263
231
|
let ratings = null;
|
|
264
232
|
if (metadata.rootDataDependencies?.includes('ratings')) {
|
|
265
|
-
try {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
|
|
269
|
-
ratings = null;
|
|
270
|
-
}
|
|
271
|
-
if (!ratings && !allowMissing) {
|
|
272
|
-
throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'ratings' is missing for ${metadata.name}`);
|
|
273
|
-
}
|
|
233
|
+
try { ratings = await loader.loadRatings(dateStr); }
|
|
234
|
+
catch (e) { if (!allowMissing) throw e; ratings = null; }
|
|
235
|
+
if (!ratings && !allowMissing) throw new Error(`Missing ratings`);
|
|
274
236
|
}
|
|
275
237
|
|
|
276
238
|
let pageViews = null;
|
|
277
239
|
if (metadata.rootDataDependencies?.includes('pageViews')) {
|
|
278
|
-
try {
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
|
|
282
|
-
pageViews = null;
|
|
283
|
-
}
|
|
284
|
-
if (!pageViews && !allowMissing) {
|
|
285
|
-
throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'pageViews' is missing for ${metadata.name}`);
|
|
286
|
-
}
|
|
240
|
+
try { pageViews = await loader.loadPageViews(dateStr); }
|
|
241
|
+
catch (e) { if (!allowMissing) throw e; pageViews = null; }
|
|
242
|
+
if (!pageViews && !allowMissing) throw new Error(`Missing pageViews`);
|
|
287
243
|
}
|
|
288
244
|
|
|
289
245
|
let watchlistMembership = null;
|
|
290
246
|
if (metadata.rootDataDependencies?.includes('watchlist')) {
|
|
291
|
-
try {
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
|
|
295
|
-
watchlistMembership = null;
|
|
296
|
-
}
|
|
297
|
-
if (!watchlistMembership && !allowMissing) {
|
|
298
|
-
throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'watchlist' is missing for ${metadata.name}`);
|
|
299
|
-
}
|
|
247
|
+
try { watchlistMembership = await loader.loadWatchlistMembership(dateStr); }
|
|
248
|
+
catch (e) { if (!allowMissing) throw e; watchlistMembership = null; }
|
|
249
|
+
if (!watchlistMembership && !allowMissing) throw new Error(`Missing watchlist`);
|
|
300
250
|
}
|
|
301
251
|
|
|
302
252
|
let alertHistory = null;
|
|
303
253
|
if (metadata.rootDataDependencies?.includes('alerts')) {
|
|
304
|
-
try {
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);
|
|
308
|
-
alertHistory = null;
|
|
309
|
-
}
|
|
310
|
-
if (!alertHistory && !allowMissing) {
|
|
311
|
-
throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'alerts' is missing for ${metadata.name}`);
|
|
312
|
-
}
|
|
254
|
+
try { alertHistory = await loader.loadAlertHistory(dateStr); }
|
|
255
|
+
catch (e) { if (!allowMissing) throw e; alertHistory = null; }
|
|
256
|
+
if (!alertHistory && !allowMissing) throw new Error(`Missing alerts`);
|
|
313
257
|
}
|
|
314
258
|
|
|
315
|
-
// [NOTE] "executeOncePerDay" is typically for sharded price/batch jobs.
|
|
316
|
-
// We initialize empty series data to maintain compatibility.
|
|
317
259
|
const seriesData = { root: {}, results: {} };
|
|
318
260
|
|
|
319
261
|
if (metadata.rootDataDependencies?.includes('price')) {
|
|
@@ -330,12 +272,10 @@ class MetaExecutor {
|
|
|
330
272
|
previousComputedDependencies: prevDeps, config, deps,
|
|
331
273
|
allRankings: rankings,
|
|
332
274
|
allRankingsYesterday: rankingsYesterday,
|
|
333
|
-
// [NEW] Pass New Root Data Types for Profile Metrics
|
|
334
275
|
ratings: ratings || {},
|
|
335
276
|
pageViews: pageViews || {},
|
|
336
277
|
watchlistMembership: watchlistMembership || {},
|
|
337
278
|
alertHistory: alertHistory || {},
|
|
338
|
-
// [NEW] Pass Series Data
|
|
339
279
|
seriesData
|
|
340
280
|
});
|
|
341
281
|
|
|
@@ -357,12 +297,10 @@ class MetaExecutor {
|
|
|
357
297
|
previousComputedDependencies: prevDeps, config, deps,
|
|
358
298
|
allRankings: rankings,
|
|
359
299
|
allRankingsYesterday: rankingsYesterday,
|
|
360
|
-
// [NEW] Pass New Root Data Types for Profile Metrics
|
|
361
300
|
ratings: ratings || {},
|
|
362
301
|
pageViews: pageViews || {},
|
|
363
302
|
watchlistMembership: watchlistMembership || {},
|
|
364
303
|
alertHistory: alertHistory || {},
|
|
365
|
-
// [NEW] Pass Series Data
|
|
366
304
|
seriesData
|
|
367
305
|
});
|
|
368
306
|
const res = await calcInstance.process(context);
|
|
@@ -4,6 +4,7 @@ const { CachedDataLoader } = require
|
|
|
4
4
|
const { ContextFactory } = require('../context/ContextFactory');
|
|
5
5
|
const { commitResults } = require('../persistence/ResultCommitter');
|
|
6
6
|
const { fetchResultSeries } = require('../data/DependencyFetcher');
|
|
7
|
+
const { getManifest } = require('../topology/ManifestLoader');
|
|
7
8
|
const mathLayer = require('../layers/index');
|
|
8
9
|
const { performance } = require('perf_hooks');
|
|
9
10
|
const v8 = require('v8');
|
|
@@ -71,6 +72,11 @@ class StandardExecutor {
|
|
|
71
72
|
|
|
72
73
|
if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
|
|
73
74
|
|
|
75
|
+
// --- 0. [NEW] Build Global Manifest Lookup ---
|
|
76
|
+
const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps);
|
|
77
|
+
const manifestLookup = {};
|
|
78
|
+
allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
|
|
79
|
+
|
|
74
80
|
// --- 1. Resolve and Filter Portfolio Refs (Today) ---
|
|
75
81
|
let effectivePortfolioRefs = portfolioRefs;
|
|
76
82
|
if (!effectivePortfolioRefs) {
|
|
@@ -177,22 +183,8 @@ class StandardExecutor {
|
|
|
177
183
|
const maxDays = Math.max(...Object.values(dependencySeriesRequests));
|
|
178
184
|
logger.log('INFO', `[StandardExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
|
|
179
185
|
|
|
180
|
-
|
|
181
|
-
const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch,
|
|
182
|
-
|
|
183
|
-
// [NEW] Validate Series Availability
|
|
184
|
-
streamingCalcs.forEach(c => {
|
|
185
|
-
if (c.manifest.dependencySeries && !c.manifest.canHaveMissingSeries) {
|
|
186
|
-
Object.keys(c.manifest.dependencySeries).forEach(depName => {
|
|
187
|
-
const normDep = normalizeName(depName);
|
|
188
|
-
// If series data is completely missing for a required dependency, verify logic
|
|
189
|
-
if (!resultsSeries[normDep] || Object.keys(resultsSeries[normDep]).length === 0) {
|
|
190
|
-
logger.log('WARN', `[StandardExecutor] Missing dependency series '${depName}' for '${c.name}' (and canHaveMissingSeries=false). This may cause calculation errors.`);
|
|
191
|
-
}
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
});
|
|
195
|
-
|
|
186
|
+
// Pass manifestLookup to fetcher
|
|
187
|
+
const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
|
|
196
188
|
seriesData.results = resultsSeries;
|
|
197
189
|
}
|
|
198
190
|
// ---------------------------------------------
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// Change this string to force a global re-computation
|
|
2
|
-
module.exports = "v2.0-epoch-
|
|
2
|
+
module.exports = "v2.0-epoch-6";
|