bulltrackers-module 1.0.161 → 1.0.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,66 +1,110 @@
|
|
|
1
1
|
const { FieldPath } = require('@google-cloud/firestore');
|
|
2
|
-
const {
|
|
2
|
+
const {
|
|
3
|
+
getPortfolioPartRefs,
|
|
4
|
+
loadFullDayMap,
|
|
5
|
+
loadDataByRefs,
|
|
6
|
+
loadDailyInsights,
|
|
7
|
+
loadDailySocialPostInsights,
|
|
8
|
+
getHistoryPartRefs,
|
|
9
|
+
streamPortfolioData,
|
|
10
|
+
streamHistoryData
|
|
11
|
+
} = require('../utils/data_loader.js');
|
|
3
12
|
const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
|
|
4
13
|
const { batchStoreSchemas } = require('../utils/schema_capture.js');
|
|
5
14
|
|
|
6
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* Stage 1: Group manifest by pass number
|
|
17
|
+
* @param {Array} manifest - The full manifest array.
|
|
18
|
+
* @returns {object} An object with pass numbers as keys and calc arrays as values.
|
|
19
|
+
*/
|
|
7
20
|
function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
|
|
8
21
|
|
|
9
|
-
/**
|
|
22
|
+
/**
|
|
10
23
|
* Stage 2: Check root data dependencies for a calc
|
|
24
|
+
* @param {object} calcManifest - A single calculation's manifest entry.
|
|
25
|
+
* @param {object} rootDataStatus - The status object from checkRootDataAvailability.
|
|
26
|
+
* @returns {{canRun: boolean, missing: string[]}}
|
|
11
27
|
*/
|
|
12
28
|
function checkRootDependencies(calcManifest, rootDataStatus) {
|
|
13
29
|
const missing = [];
|
|
14
|
-
if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) { return { canRun: true, missing };}
|
|
30
|
+
if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) { return { canRun: true, missing }; }
|
|
15
31
|
for (const dep of calcManifest.rootDataDependencies) {
|
|
16
32
|
if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
|
|
17
33
|
else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
|
|
18
34
|
else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
|
|
19
|
-
else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
|
|
35
|
+
else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
|
|
36
|
+
}
|
|
20
37
|
return { canRun: missing.length === 0, missing };
|
|
21
|
-
}
|
|
38
|
+
}
|
|
22
39
|
|
|
23
|
-
/**
|
|
40
|
+
/**
|
|
24
41
|
* Stage 3: Check root data availability for a date
|
|
42
|
+
* @param {string} dateStr - The date to check (YYYY-MM-DD).
|
|
43
|
+
* @param {object} config - The computation system config.
|
|
44
|
+
* @param {object} dependencies - All shared dependencies (db, logger, etc.).
|
|
45
|
+
* @param {object} earliestDates - The map of earliest data dates.
|
|
46
|
+
* @returns {Promise<object|null>} The root data object or null.
|
|
25
47
|
*/
|
|
26
48
|
async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
|
|
27
49
|
const { logger } = dependencies;
|
|
28
50
|
logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
|
|
29
51
|
const dateToProcess = new Date(dateStr + 'T00:00:00Z');
|
|
52
|
+
|
|
30
53
|
let portfolioRefs = [], insightsData = null, socialData = null, historyRefs = [];
|
|
31
54
|
let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
|
|
55
|
+
|
|
32
56
|
try {
|
|
33
57
|
const tasks = [];
|
|
34
|
-
if
|
|
35
|
-
if (dateToProcess >= earliestDates.
|
|
36
|
-
|
|
37
|
-
|
|
58
|
+
// Only query for data if the processing date is on or after the earliest possible date
|
|
59
|
+
if (dateToProcess >= earliestDates.portfolio) {
|
|
60
|
+
tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => { portfolioRefs = res; hasPortfolio = !!(res?.length); }));
|
|
61
|
+
}
|
|
62
|
+
if (dateToProcess >= earliestDates.insights) {
|
|
63
|
+
tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => { insightsData = res; hasInsights = !!res; }));
|
|
64
|
+
}
|
|
65
|
+
if (dateToProcess >= earliestDates.social) {
|
|
66
|
+
tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => { socialData = res; hasSocial = !!res; }));
|
|
67
|
+
}
|
|
68
|
+
if (dateToProcess >= earliestDates.history) {
|
|
69
|
+
tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => { historyRefs = res; hasHistory = !!(res?.length); }));
|
|
70
|
+
}
|
|
71
|
+
|
|
38
72
|
await Promise.all(tasks);
|
|
39
|
-
|
|
73
|
+
|
|
74
|
+
logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
|
|
75
|
+
|
|
40
76
|
if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
|
|
41
|
-
|
|
42
|
-
|
|
77
|
+
|
|
78
|
+
return {
|
|
79
|
+
portfolioRefs,
|
|
80
|
+
todayInsights: insightsData,
|
|
81
|
+
todaySocialPostInsights: socialData,
|
|
82
|
+
historyRefs,
|
|
83
|
+
status: { hasPortfolio, hasInsights, hasSocial, hasHistory }
|
|
84
|
+
};
|
|
85
|
+
} catch (err) {
|
|
86
|
+
logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message });
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
43
89
|
}
|
|
44
90
|
|
|
45
91
|
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* This resolves the bug where Pass 2 would run but could not find
|
|
54
|
-
* the Pass 1 results it depended on.
|
|
92
|
+
* Stage 4: Fetch ALL existing computed results for the pass AND their dependencies.
|
|
93
|
+
* @param {string} dateStr - The date to fetch (YYYY-MM-DD).
|
|
94
|
+
* @param {Array} calcsInPass - The calculations in the *current* pass.
|
|
95
|
+
* @param {Array} fullManifest - The *entire* computation manifest.
|
|
96
|
+
* @param {object} config - The computation system config.
|
|
97
|
+
* @param {object} dependencies - Shared dependencies.
|
|
98
|
+
* @returns {Promise<object>} A map of { [calcName]: result } for all found dependencies.
|
|
55
99
|
*/
|
|
56
100
|
async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
|
|
57
101
|
const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
|
|
58
102
|
const calcsToFetch = new Set();
|
|
59
103
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
calcsToFetch.add(calcName);
|
|
104
|
+
// Add all calcs in this pass (to check for completion)
|
|
105
|
+
for (const calc of calcsInPass) { calcsToFetch.add(normalizeName(calc.name));
|
|
63
106
|
|
|
107
|
+
// Add all dependencies of those calcs (for meta-calcs)
|
|
64
108
|
if (calc.dependencies && calc.dependencies.length > 0) {
|
|
65
109
|
for (const depName of calc.dependencies) {
|
|
66
110
|
calcsToFetch.add(normalizeName(depName));
|
|
@@ -68,22 +112,16 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
|
|
|
68
112
|
}
|
|
69
113
|
}
|
|
70
114
|
|
|
71
|
-
if (!calcsToFetch.size) {
|
|
72
|
-
return {};
|
|
73
|
-
}
|
|
115
|
+
if (!calcsToFetch.size) { return {}; }
|
|
74
116
|
|
|
75
117
|
logger.log('INFO', `[PassRunner] Checking for ${calcsToFetch.size} existing results and dependencies for ${dateStr}...`);
|
|
76
118
|
|
|
77
119
|
const docRefs = [];
|
|
78
120
|
const depNames = [];
|
|
79
121
|
|
|
80
|
-
// --- FIX: Iterate the Set of all calcs to fetch ---
|
|
81
122
|
for (const calcName of calcsToFetch) {
|
|
82
123
|
const calcManifest = manifestMap.get(calcName);
|
|
83
|
-
if (!calcManifest) {
|
|
84
|
-
logger.log('ERROR', `[PassRunner] Missing manifest for dependency: ${calcName}`);
|
|
85
|
-
continue;
|
|
86
|
-
}
|
|
124
|
+
if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for dependency: ${calcName} on ${dateStr}`); continue; }
|
|
87
125
|
|
|
88
126
|
docRefs.push(
|
|
89
127
|
db.collection(config.resultsCollection)
|
|
@@ -98,308 +136,602 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
|
|
|
98
136
|
|
|
99
137
|
const fetched = {};
|
|
100
138
|
if (docRefs.length) {
|
|
101
|
-
|
|
102
|
-
|
|
139
|
+
const snapshots = await db.getAll(...docRefs);
|
|
140
|
+
snapshots.forEach((doc, i) => {
|
|
103
141
|
const data = doc.exists ? doc.data() : null;
|
|
104
|
-
if
|
|
105
|
-
|
|
106
|
-
} else {
|
|
107
|
-
fetched[depNames[i]] = null; // Treat as not existing if incomplete
|
|
142
|
+
// A result only "exists" if it was marked as completed
|
|
143
|
+
if (data && data._completed === true) { fetched[depNames[i]] = data;
|
|
144
|
+
} else { fetched[depNames[i]] = null; // Treat as not existing
|
|
108
145
|
}
|
|
109
146
|
});
|
|
110
147
|
}
|
|
111
148
|
|
|
112
|
-
//
|
|
149
|
+
// Verbose logging for what was found/missing
|
|
113
150
|
const foundDeps = Object.entries(fetched).filter(([, data]) => data !== null).map(([key]) => key);
|
|
114
151
|
const missingDeps = Object.entries(fetched).filter(([, data]) => data === null).map(([key]) => key);
|
|
115
|
-
if (foundDeps.length > 0) {
|
|
116
|
-
|
|
117
|
-
}
|
|
118
|
-
if (missingDeps.length > 0) {
|
|
119
|
-
logger.log('TRACE', `[PassRunner] Did not find ${missingDeps.length} results: [${missingDeps.join(', ')}]`);
|
|
120
|
-
}
|
|
152
|
+
if (foundDeps.length > 0) { logger.log('TRACE', `[PassRunner] Found ${foundDeps.length} existing results for ${dateStr}: [${foundDeps.join(', ')}]`); }
|
|
153
|
+
if (missingDeps.length > 0) { logger.log('TRACE', `[PassRunner] Did not find ${missingDeps.length} results for ${dateStr}: [${missingDeps.join(', ')}]`); }
|
|
121
154
|
|
|
122
155
|
return fetched;
|
|
123
156
|
}
|
|
124
157
|
|
|
125
|
-
|
|
126
158
|
/**
|
|
127
|
-
*
|
|
128
|
-
* This function now implements your "even better design".
|
|
129
|
-
* It calculates the *true earliest run date* for every calculation
|
|
130
|
-
* and filters them out *before* the "Running..." log ever appears.
|
|
159
|
+
* Stage 5: Filter calculations based on data availability and completion status.
|
|
131
160
|
*/
|
|
132
161
|
function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates, logger) {
|
|
133
162
|
const skipped = new Set();
|
|
134
163
|
const dateToProcess = new Date(dateStr + 'T00:00:00Z');
|
|
164
|
+
|
|
165
|
+
// Helper to find the true earliest date a calc can run
|
|
135
166
|
const getTrueEarliestRunDate = (calc) => {
|
|
136
|
-
let earliestRunDate = new Date('1970-01-01T00:00:00Z');
|
|
167
|
+
let earliestRunDate = new Date('1970-01-01T00:00:00Z');
|
|
137
168
|
const dependencies = calc.rootDataDependencies || [];
|
|
169
|
+
|
|
138
170
|
for (const dep of dependencies) {
|
|
139
171
|
if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
|
|
140
172
|
if (dep === 'history' && earliestDates.history > earliestRunDate) earliestRunDate = earliestDates.history;
|
|
141
173
|
if (dep === 'social' && earliestDates.social > earliestRunDate) earliestRunDate = earliestDates.social;
|
|
142
174
|
if (dep === 'insights' && earliestDates.insights > earliestRunDate) earliestRunDate = earliestDates.insights;
|
|
143
175
|
}
|
|
176
|
+
|
|
177
|
+
// If it's historical, it needs T-1 data, so add one day
|
|
144
178
|
if (calc.isHistorical && earliestRunDate.getTime() > 0) { earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1); }
|
|
145
179
|
return earliestRunDate;
|
|
146
180
|
};
|
|
181
|
+
|
|
147
182
|
const filterCalc = (calc) => {
|
|
148
|
-
//
|
|
149
|
-
if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists (and is complete).`);
|
|
183
|
+
// 1. Skip if already completed
|
|
184
|
+
if (existingResults[calc.name]) { logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists (and is complete).`);
|
|
185
|
+
skipped.add(calc.name);
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
150
188
|
|
|
189
|
+
// 2. Skip if date is before this calc's earliest possible run date
|
|
151
190
|
const earliestRunDate = getTrueEarliestRunDate(calc);
|
|
152
|
-
if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
|
|
191
|
+
if (dateToProcess < earliestRunDate) { logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
|
|
192
|
+
skipped.add(calc.name);
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// 3. Skip if missing root data
|
|
153
197
|
const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
|
|
154
|
-
if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
|
|
198
|
+
if (!canRun) { logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
|
|
199
|
+
skipped.add(calc.name);
|
|
200
|
+
return false;
|
|
201
|
+
}
|
|
155
202
|
|
|
156
|
-
|
|
203
|
+
// 4. (Meta Calcs) Skip if missing computed dependencies
|
|
204
|
+
if (calc.type === 'meta') {
|
|
157
205
|
const missingDeps = (calc.dependencies || [])
|
|
158
206
|
.map(normalizeName)
|
|
159
207
|
.filter(d => !existingResults[d]); // This check is now robust
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
skipped.add(calc.name);
|
|
208
|
+
|
|
209
|
+
if (missingDeps.length > 0) { logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
|
|
210
|
+
skipped.add(calc.name);
|
|
163
211
|
return false;
|
|
164
|
-
}
|
|
212
|
+
}
|
|
165
213
|
}
|
|
166
214
|
return true;
|
|
167
215
|
};
|
|
216
|
+
|
|
168
217
|
const standardCalcsToRun = standardCalcs.filter(filterCalc);
|
|
169
218
|
const metaCalcsToRun = metaCalcs.filter(filterCalc);
|
|
219
|
+
|
|
170
220
|
return { standardCalcsToRun, metaCalcsToRun };
|
|
171
221
|
}
|
|
172
222
|
|
|
173
|
-
/**
|
|
174
|
-
|
|
223
|
+
/**
|
|
224
|
+
* Stage 6: Initialize calculator instances
|
|
225
|
+
* @param {Array} calcs - The calculations to initialize.
|
|
226
|
+
* @param {object} logger - The logger instance.
|
|
227
|
+
* @returns {object} A map of { [calcName]: instance }.
|
|
228
|
+
*/
|
|
229
|
+
function initializeCalculators(calcs, logger) {
|
|
230
|
+
const state = {};
|
|
231
|
+
for (const c of calcs) {
|
|
232
|
+
const name = normalizeName(c.name);
|
|
233
|
+
const Cl = c.class;
|
|
234
|
+
if (typeof Cl === 'function') {
|
|
235
|
+
try {
|
|
236
|
+
const inst = new Cl();
|
|
237
|
+
inst.manifest = c; // Attach manifest for context
|
|
238
|
+
state[name] = inst;
|
|
239
|
+
} catch (e) { logger.warn(`Initialization failed for ${name}`, { errorMessage: e.message });
|
|
240
|
+
state[name] = null;
|
|
241
|
+
}
|
|
242
|
+
} else { logger.warn(`Class is missing for ${name}`);
|
|
243
|
+
state[name] = null;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return state;
|
|
247
|
+
}
|
|
175
248
|
|
|
176
|
-
/**
|
|
249
|
+
/**
|
|
250
|
+
* Stage 7: Load T-1 (yesterday) data needed by historical calculations.
|
|
251
|
+
* --- THIS FUNCTION IS NOW FIXED ---
|
|
177
252
|
*/
|
|
178
|
-
async function loadHistoricalData(date, calcs, config, deps, rootData) {
|
|
179
|
-
const { logger }
|
|
180
|
-
const updated
|
|
181
|
-
const tasks
|
|
182
|
-
|
|
183
|
-
|
|
253
|
+
async function loadHistoricalData(date, calcs, config, deps, rootData) {
|
|
254
|
+
const { logger } = deps;
|
|
255
|
+
const updated = { ...rootData };
|
|
256
|
+
const tasks = [];
|
|
257
|
+
|
|
258
|
+
// Check what T-1 data is needed by any calc in this pass
|
|
259
|
+
const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
|
|
260
|
+
const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
|
|
184
261
|
const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
|
|
185
|
-
|
|
186
|
-
// --- FIX: Add T-1 COMPUTED dependency loading ---
|
|
262
|
+
const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history')); // <-- THE FIX
|
|
187
263
|
const needsYesterdayDependencies = calcs.some(c => c.isHistorical && c.dependencies && c.dependencies.length > 0);
|
|
188
|
-
|
|
264
|
+
|
|
189
265
|
const prev = new Date(date);
|
|
190
266
|
prev.setUTCDate(prev.getUTCDate() - 1);
|
|
191
267
|
const prevStr = prev.toISOString().slice(0, 10);
|
|
192
|
-
|
|
193
|
-
if(needsYesterdayInsights) {
|
|
194
|
-
tasks.push((async()=>{
|
|
195
|
-
|
|
196
|
-
updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); })());}
|
|
197
|
-
if(needsYesterdaySocial) {
|
|
198
|
-
tasks.push((async()=>{
|
|
199
|
-
logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
|
|
200
|
-
updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
|
|
201
|
-
|
|
202
|
-
if(needsYesterdayPortfolio) {
|
|
203
|
-
tasks.push((async()=>{
|
|
204
|
-
logger.log('INFO', `[PassRunner] Getting YESTERDAY portfolio refs for ${prevStr}`);
|
|
205
|
-
updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
|
|
268
|
+
|
|
269
|
+
if (needsYesterdayInsights) {
|
|
270
|
+
tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
|
|
271
|
+
updated.yesterdayInsights = await loadDailyInsights(config, deps, prevStr);
|
|
206
272
|
})());
|
|
207
273
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
274
|
+
if (needsYesterdaySocial) {
|
|
275
|
+
tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
|
|
276
|
+
updated.yesterdaySocialPostInsights = await loadDailySocialPostInsights(config, deps, prevStr);
|
|
277
|
+
})());
|
|
278
|
+
}
|
|
279
|
+
if (needsYesterdayPortfolio) {
|
|
280
|
+
tasks.push((async () => { logger.log('INFO', `[PassRunner] Getting YESTERDAY portfolio refs for ${prevStr}`);
|
|
281
|
+
updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
|
|
282
|
+
})());
|
|
283
|
+
}
|
|
284
|
+
// --- FIX: Load yesterday's history refs ---
|
|
285
|
+
if (needsYesterdayHistory) {
|
|
286
|
+
tasks.push((async () => { logger.log('INFO', `[PassRunner] Getting YESTERDAY history refs for ${prevStr}`);
|
|
287
|
+
updated.yesterdayHistoryRefs = await getHistoryPartRefs(config, deps, prevStr);
|
|
288
|
+
})());
|
|
289
|
+
}
|
|
290
|
+
// --- END FIX ---
|
|
291
|
+
|
|
292
|
+
if (needsYesterdayDependencies) {
|
|
293
|
+
tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY computed dependencies for ${prevStr}`);
|
|
294
|
+
// This fetches T-1 results for *all* calcs in the current pass,
|
|
295
|
+
// which is robust and covers all historical dependency needs.
|
|
215
296
|
updated.yesterdayDependencyData = await fetchExistingResults(prevStr, calcs, calcs.map(c => c.manifest), config, deps);
|
|
216
297
|
})());
|
|
217
298
|
}
|
|
218
299
|
|
|
219
|
-
await Promise.all(tasks);
|
|
220
|
-
return updated;
|
|
300
|
+
await Promise.all(tasks);
|
|
301
|
+
return updated;
|
|
221
302
|
}
|
|
222
303
|
|
|
223
304
|
/**
|
|
224
|
-
*
|
|
305
|
+
* Stage 8: Stream and process data for standard calculations.
|
|
306
|
+
* --- THIS FUNCTION IS NOW FIXED ---
|
|
225
307
|
*/
|
|
226
|
-
async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) {
|
|
308
|
+
async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) {
|
|
227
309
|
const { logger, calculationUtils } = deps;
|
|
228
310
|
const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, yesterdayDependencyData } = rootData;
|
|
229
|
-
const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
|
|
230
311
|
|
|
231
|
-
//
|
|
232
|
-
const
|
|
312
|
+
// Create the shared context object
|
|
313
|
+
const mappings = await calculationUtils.loadInstrumentMappings();
|
|
314
|
+
const context = {
|
|
315
|
+
instrumentMappings: mappings.instrumentToTicker,
|
|
316
|
+
sectorMapping: mappings.instrumentToSector,
|
|
317
|
+
todayDateStr: dateStr,
|
|
318
|
+
dependencies: deps,
|
|
319
|
+
config,
|
|
320
|
+
yesterdaysDependencyData: yesterdayDependencyData
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
// --- Run non-streaming (meta) calcs once ---
|
|
324
|
+
let firstUser = true;
|
|
325
|
+
for (const name in state) {
|
|
326
|
+
const calc = state[name];
|
|
327
|
+
if (!calc || typeof calc.process !== 'function') continue;
|
|
328
|
+
|
|
329
|
+
const cat = calc.manifest.category;
|
|
330
|
+
if (cat === 'socialPosts' || cat === 'insights') {
|
|
331
|
+
if (firstUser) {
|
|
332
|
+
logger.log('INFO', `[${passName}] Running non-streaming calc: ${name} for ${dateStr}`);
|
|
333
|
+
const args = [null, null, null, { ...context, userType: 'n/a' }, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, null, null];
|
|
334
|
+
try {
|
|
335
|
+
await Promise.resolve(calc.process(...args));
|
|
336
|
+
} catch (e) { logger.log('WARN', `Process error on ${name} (non-stream) for ${dateStr}`, { err: e.message }); }
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const calcsThatStreamPortfolio = Object.values(state).filter(calc =>
|
|
342
|
+
calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio'))
|
|
343
|
+
);
|
|
344
|
+
|
|
345
|
+
if (calcsThatStreamPortfolio.length === 0) {
|
|
346
|
+
logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs for ${dateStr}...`);
|
|
233
351
|
|
|
234
|
-
let firstUser=true;
|
|
235
|
-
for(const name in state){ const calc=state[name]; if(!calc||typeof calc.process!=='function') continue;
|
|
236
|
-
const cat=calc.manifest.category;
|
|
237
|
-
if(cat==='socialPosts'||cat==='insights') {
|
|
238
|
-
if (firstUser) {
|
|
239
|
-
logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
|
|
240
|
-
let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
|
|
241
|
-
if(calc.manifest.isHistorical) { args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null]; }
|
|
242
|
-
try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});} } } }
|
|
243
|
-
if (calcsThatStreamPortfolio.length === 0) { logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`); return; }
|
|
244
|
-
logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs...`);
|
|
245
352
|
const prevDate = new Date(dateStr + 'T00:00:00Z');
|
|
246
353
|
prevDate.setUTCDate(prevDate.getUTCDate() - 1);
|
|
247
354
|
const prevDateStr = prevDate.toISOString().slice(0, 10);
|
|
355
|
+
|
|
356
|
+
// Check which iterators we need
|
|
248
357
|
const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
|
|
249
358
|
const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
|
|
359
|
+
const needsYesterdayHistory = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('history')); // <-- THE FIX
|
|
360
|
+
|
|
361
|
+
// --- Create all necessary iterators ---
|
|
362
|
+
const tP_iterator = streamPortfolioData(config, deps, dateStr, portfolioRefs);
|
|
250
363
|
const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
|
|
251
364
|
const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
|
|
365
|
+
const hY_iterator = needsYesterdayHistory ? streamHistoryData(config, deps, prevDateStr, rootData.yesterdayHistoryRefs) : null; // <-- THE FIX
|
|
366
|
+
|
|
252
367
|
let yesterdayPortfolios = {};
|
|
253
368
|
let todayHistoryData = {};
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
if (yP_iterator) {
|
|
258
|
-
if (hT_iterator) {
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
369
|
+
let yesterdayHistoryData = {}; // <-- THE FIX
|
|
370
|
+
|
|
371
|
+
// Pre-load the first chunk of historical data
|
|
372
|
+
if (yP_iterator) { Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
|
|
373
|
+
if (hT_iterator) { Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); }
|
|
374
|
+
if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); } // <-- THE FIX
|
|
375
|
+
|
|
376
|
+
for await (const chunk of tP_iterator) {
|
|
377
|
+
// Load the *next* chunk of historical data to stay in sync
|
|
378
|
+
if (yP_iterator) { Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
|
|
379
|
+
if (hT_iterator) { Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); }
|
|
380
|
+
if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); } // <-- THE FIX
|
|
381
|
+
|
|
382
|
+
for (const uid in chunk) {
|
|
383
|
+
const p = chunk[uid]; // Today's Portfolio
|
|
384
|
+
if (!p) continue;
|
|
385
|
+
|
|
386
|
+
const userType = p.PublicPositions ? 'speculator' : 'normal';
|
|
387
|
+
context.userType = userType;
|
|
388
|
+
|
|
389
|
+
// Get corresponding T-1 data
|
|
390
|
+
const pY = yesterdayPortfolios[uid] || null;
|
|
391
|
+
const hT = todayHistoryData[uid] || null;
|
|
392
|
+
const hY = yesterdayHistoryData[uid] || null; // <-- THE FIX
|
|
393
|
+
|
|
394
|
+
for (const name in state) {
|
|
395
|
+
const calc = state[name];
|
|
396
|
+
if (!calc || typeof calc.process !== 'function') continue;
|
|
397
|
+
|
|
398
|
+
// --- Refactored Filter Block ---
|
|
399
|
+
const manifest = calc.manifest;
|
|
400
|
+
const cat = manifest.category;
|
|
401
|
+
const isSocialOrInsights = cat === 'socialPosts' || cat === 'insights';
|
|
402
|
+
if (isSocialOrInsights) continue; // Handled above
|
|
403
|
+
|
|
404
|
+
const isSpeculatorCalc = cat === 'speculators';
|
|
405
|
+
const isUserProcessed = name === 'users-processed';
|
|
406
|
+
|
|
407
|
+
// Skip if user type doesn't match calc type
|
|
408
|
+
if (userType === 'normal' && isSpeculatorCalc) continue;
|
|
409
|
+
if (userType === 'speculator' && !isSpeculatorCalc && !isUserProcessed) continue;
|
|
410
|
+
|
|
411
|
+
// Skip historical calcs if T-1 portfolio is missing (with exceptions)
|
|
412
|
+
if (manifest.isHistorical && !pY) {
|
|
413
|
+
// These calcs only need T-1 *history*, not portfolio
|
|
414
|
+
if (cat !== 'behavioural' && name !== 'historical-performance-aggregator') {
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
// --- End Filter Block ---
|
|
419
|
+
|
|
420
|
+
let args;
|
|
421
|
+
if (manifest.isHistorical) {
|
|
422
|
+
// Full 10-argument array for historical calcs
|
|
423
|
+
args = [p, pY, uid, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, hT, hY]; // <-- hY is 10th arg
|
|
424
|
+
} else {
|
|
425
|
+
// 10-argument array for non-historical (T-1 args are null)
|
|
426
|
+
args = [p, null, uid, context, todayInsights, null, todaySocialPostInsights, null, hT, null];
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
try {
|
|
430
|
+
await Promise.resolve(calc.process(...args));
|
|
431
|
+
} catch (e) {
|
|
432
|
+
logger.log('WARN', `Process error on ${name} for ${uid} on ${dateStr}`, { err: e.message });
|
|
433
|
+
}
|
|
434
|
+
} // end for(calc)
|
|
435
|
+
|
|
436
|
+
firstUser = false;
|
|
437
|
+
|
|
438
|
+
// Clear processed users from memory
|
|
439
|
+
if (pY) { delete yesterdayPortfolios[uid]; }
|
|
440
|
+
if (hT) { delete todayHistoryData[uid]; }
|
|
441
|
+
if (hY) { delete yesterdayHistoryData[uid]; }
|
|
442
|
+
} // end for(uid in chunk)
|
|
443
|
+
} // end for await(chunk)
|
|
444
|
+
|
|
445
|
+
// Clear stale data to prevent memory leaks
|
|
278
446
|
yesterdayPortfolios = {};
|
|
279
447
|
todayHistoryData = {};
|
|
448
|
+
yesterdayHistoryData = {};
|
|
280
449
|
|
|
281
450
|
logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
|
|
282
451
|
}
|
|
283
452
|
|
|
284
|
-
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Stage 9: Run standard computations
|
|
456
|
+
* @param {Date} date - The date to run for.
|
|
457
|
+
* @param {Array} calcs - The calculations to run.
|
|
458
|
+
* @param {string} passName - The name of the pass (for logging).
|
|
459
|
+
* @param {object} config - Computation system config.
|
|
460
|
+
* @param {object} deps - Shared dependencies.
|
|
461
|
+
* @param {object} rootData - The loaded root data for today.
|
|
462
|
+
*/
|
|
285
463
|
async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
|
|
286
464
|
const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
|
|
287
|
-
if (calcs.length === 0) {
|
|
288
|
-
|
|
465
|
+
if (calcs.length === 0) {
|
|
466
|
+
logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
|
|
467
|
+
return;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} standard calcs: [${calcs.map(c => c.name).join(', ')}]`);
|
|
471
|
+
|
|
472
|
+
// Load T-1 data (portfolio, insights, social, history, computed)
|
|
289
473
|
const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
|
|
474
|
+
|
|
475
|
+
// Initialize calcs
|
|
290
476
|
const state = initializeCalculators(calcs, logger);
|
|
477
|
+
|
|
478
|
+
// Stream T and T-1 data and process
|
|
291
479
|
await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs);
|
|
292
|
-
|
|
480
|
+
|
|
481
|
+
// --- Verbose Logging Setup ---
|
|
482
|
+
const successCalcs = [];
|
|
293
483
|
const failedCalcs = [];
|
|
484
|
+
|
|
294
485
|
const standardWrites = [];
|
|
295
|
-
const shardedWrites = {};
|
|
486
|
+
const shardedWrites = {};
|
|
296
487
|
const schemasToStore = [];
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
488
|
+
|
|
489
|
+
// --- Get Results ---
|
|
490
|
+
for (const name in state) {
|
|
491
|
+
const calc = state[name];
|
|
492
|
+
if (!calc || typeof calc.getResult !== 'function') continue;
|
|
493
|
+
|
|
494
|
+
try {
|
|
495
|
+
const result = await Promise.resolve(calc.getResult());
|
|
496
|
+
|
|
497
|
+
if (result && Object.keys(result).length > 0) {
|
|
498
|
+
const standardResult = {};
|
|
499
|
+
|
|
500
|
+
// --- Handle Sharded Writes ---
|
|
501
|
+
for (const key in result) {
|
|
502
|
+
if (key.startsWith('sharded_')) {
|
|
503
|
+
const shardedData = result[key];
|
|
504
|
+
for (const collectionName in shardedData) {
|
|
505
|
+
if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
|
|
506
|
+
Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
|
|
507
|
+
}
|
|
508
|
+
} else {
|
|
509
|
+
standardResult[key] = result[key];
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// --- Handle Standard Writes ---
|
|
514
|
+
if (Object.keys(standardResult).length > 0) {
|
|
515
|
+
const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
|
|
516
|
+
.collection(config.resultsSubcollection).doc(calc.manifest.category)
|
|
517
|
+
.collection(config.computationsSubcollection).doc(name);
|
|
518
|
+
|
|
519
|
+
standardResult._completed = true; // Mark as complete
|
|
520
|
+
standardWrites.push({ ref: docRef, data: standardResult });
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// --- Capture Schema ---
|
|
524
|
+
const calcClass = calc.manifest.class;
|
|
525
|
+
let staticSchema = null;
|
|
526
|
+
if (calcClass && typeof calcClass.getSchema === 'function') {
|
|
527
|
+
try {
|
|
528
|
+
staticSchema = calcClass.getSchema();
|
|
529
|
+
} catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name} on ${dStr}`, { err: e.message }); }
|
|
530
|
+
} else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
|
|
531
|
+
|
|
532
|
+
if (staticSchema) {
|
|
533
|
+
schemasToStore.push({
|
|
534
|
+
name,
|
|
535
|
+
category: calc.manifest.category,
|
|
536
|
+
schema: staticSchema,
|
|
537
|
+
metadata: {
|
|
538
|
+
isHistorical: calc.manifest.isHistorical || false,
|
|
539
|
+
dependencies: calc.manifest.dependencies || [],
|
|
540
|
+
rootDataDependencies: calc.manifest.rootDataDependencies || [],
|
|
541
|
+
pass: calc.manifest.pass,
|
|
542
|
+
type: calc.manifest.type || 'standard'
|
|
543
|
+
}
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
successCalcs.push(name);
|
|
547
|
+
} else {
|
|
548
|
+
// Calc ran but returned no data
|
|
549
|
+
successCalcs.push(name);
|
|
550
|
+
}
|
|
551
|
+
} catch (e) { logger.log('ERROR', `getResult failed for ${name} on ${dStr}`, { err: e.message, stack: e.stack });
|
|
552
|
+
failedCalcs.push({ name, error: e.message });
|
|
553
|
+
}
|
|
554
|
+
} // --- End Get Results Loop ---
|
|
555
|
+
|
|
556
|
+
// --- Commit Writes ---
|
|
557
|
+
if (schemasToStore.length > 0) {
|
|
558
|
+
batchStoreSchemas(deps, config, schemasToStore).catch(err => { logger.log('WARN', `[SchemaCapture] Non-blocking schema storage failed for ${dStr}`, { errorMessage: err.message }); }); }
|
|
559
|
+
|
|
323
560
|
if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`); }
|
|
324
|
-
|
|
325
|
-
const
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
561
|
+
|
|
562
|
+
for (const docPath in shardedWrites) {
|
|
563
|
+
const docData = shardedWrites[docPath];
|
|
564
|
+
const shardedDocWrites = [];
|
|
565
|
+
let docRef;
|
|
566
|
+
if (docPath.includes('/')) { docRef = deps.db.doc(docPath);
|
|
567
|
+
} else { const collection = (docPath.startsWith('user_profile_history')) ? config.shardedUserProfileCollection : config.shardedProfitabilityCollection; docRef = deps.db.collection(collection).doc(docPath); }
|
|
568
|
+
|
|
569
|
+
if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
|
|
570
|
+
docData._completed = true;
|
|
571
|
+
shardedDocWrites.push({ ref: docRef, data: docData });
|
|
572
|
+
} else { logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath} on ${dStr}. Not an object.`, { data: docData }); }
|
|
573
|
+
|
|
574
|
+
if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`); }
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// --- Final Verbose Log ---
|
|
578
|
+
const logMetadata = {
|
|
579
|
+
total_expected: calcs.length,
|
|
580
|
+
success_count: successCalcs.length,
|
|
581
|
+
failed_count: failedCalcs.length,
|
|
582
|
+
successful_calcs: successCalcs,
|
|
583
|
+
failed_calcs: failedCalcs
|
|
584
|
+
};
|
|
585
|
+
logger.log( failedCalcs.length === 0 ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}.`, logMetadata );
|
|
340
586
|
}
|
|
341
587
|
|
|
342
588
|
/**
|
|
343
|
-
*
|
|
589
|
+
* Stage 10: Run meta computations
|
|
590
|
+
* @param {Date} date - The date to run for.
|
|
591
|
+
* @param {Array} calcs - The meta calculations to run.
|
|
592
|
+
* @param {string} passName - The name of the pass (for logging).
|
|
593
|
+
* @param {object} config - Computation system config.
|
|
594
|
+
* @param {object} deps - Shared dependencies.
|
|
595
|
+
* @param {object} fetchedDeps - In-memory results from *previous* passes.
|
|
596
|
+
* @param {object} rootData - The loaded root data for today.
|
|
344
597
|
*/
|
|
345
598
|
async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
|
|
346
599
|
const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
|
|
347
600
|
if (calcs.length === 0) { logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`); return; }
|
|
348
|
-
|
|
601
|
+
|
|
602
|
+
logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} meta calcs: [${calcs.map(c => c.name).join(', ')}]`);
|
|
603
|
+
|
|
604
|
+
// Load T-1 data (needed for stateful meta calcs)
|
|
349
605
|
const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
|
|
350
|
-
|
|
606
|
+
|
|
607
|
+
// --- Verbose Logging Setup ---
|
|
608
|
+
const successCalcs = [];
|
|
351
609
|
const failedCalcs = [];
|
|
610
|
+
|
|
352
611
|
const standardWrites = [];
|
|
353
|
-
const shardedWrites = {};
|
|
612
|
+
const shardedWrites = {};
|
|
354
613
|
const schemasToStore = [];
|
|
355
|
-
for (const mCalc of calcs) {
|
|
356
|
-
const name = normalizeName(mCalc.name), Cl = mCalc.class;
|
|
357
|
-
if (typeof Cl !== 'function') { logger.log('ERROR', `Invalid class ${name}`); failedCalcs.push(name); continue; }
|
|
358
|
-
const inst = new Cl();
|
|
359
|
-
|
|
360
|
-
try { if (typeof inst.process !== 'function') { logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`); failedCalcs.push(name); continue; }
|
|
361
614
|
|
|
362
|
-
|
|
615
|
+
for (const mCalc of calcs) {
|
|
616
|
+
const name = normalizeName(mCalc.name);
|
|
617
|
+
const Cl = mCalc.class;
|
|
618
|
+
|
|
619
|
+
if (typeof Cl !== 'function') {
|
|
620
|
+
logger.log('ERROR', `Invalid class ${name} on ${dStr}`);
|
|
621
|
+
failedCalcs.push({ name, error: "Invalid class" });
|
|
622
|
+
continue;
|
|
623
|
+
}
|
|
363
624
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
625
|
+
const inst = new Cl();
|
|
626
|
+
|
|
627
|
+
try {
|
|
628
|
+
if (typeof inst.process !== 'function') {
|
|
629
|
+
logger.log('ERROR', `Meta-calc ${name} is missing a 'process' method.`);
|
|
630
|
+
failedCalcs.push({ name, error: "Missing process method" });
|
|
631
|
+
continue;
|
|
370
632
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
633
|
+
|
|
634
|
+
// Meta-calc `process` is different: it receives the date, full dependencies, config, and fetched dependencies.
|
|
635
|
+
// It *also* gets the T-1 root data via the `dependencies.rootData` object.
|
|
636
|
+
const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
|
|
637
|
+
|
|
638
|
+
if (result && Object.keys(result).length > 0) {
|
|
639
|
+
const standardResult = {};
|
|
640
|
+
|
|
641
|
+
// --- Handle Sharded Writes ---
|
|
642
|
+
for (const key in result) {
|
|
643
|
+
if (key.startsWith('sharded_')) {
|
|
644
|
+
const shardedData = result[key];
|
|
645
|
+
for (const collectionName in shardedData) {
|
|
646
|
+
if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
|
|
647
|
+
Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
|
|
648
|
+
}
|
|
649
|
+
} else {
|
|
650
|
+
standardResult[key] = result[key];
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// --- Handle Standard Writes ---
|
|
655
|
+
if (Object.keys(standardResult).length > 0) {
|
|
656
|
+
const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
|
|
657
|
+
.collection(config.resultsSubcollection).doc(mCalc.category)
|
|
658
|
+
.collection(config.computationsSubcollection).doc(name);
|
|
659
|
+
|
|
660
|
+
standardResult._completed = true; // Mark as complete
|
|
661
|
+
standardWrites.push({ ref: docRef, data: standardResult });
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// --- Capture Schema ---
|
|
665
|
+
const calcClass = mCalc.class;
|
|
666
|
+
let staticSchema = null;
|
|
667
|
+
if (calcClass && typeof calcClass.getSchema === 'function') {
|
|
668
|
+
try {
|
|
669
|
+
staticSchema = calcClass.getSchema();
|
|
670
|
+
} catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name} on ${dStr}`, { err: e.message }); }
|
|
671
|
+
} else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
|
|
672
|
+
|
|
673
|
+
if (staticSchema) {
|
|
674
|
+
schemasToStore.push({
|
|
675
|
+
name,
|
|
676
|
+
category: mCalc.category,
|
|
677
|
+
schema: staticSchema,
|
|
678
|
+
metadata: {
|
|
679
|
+
isHistorical: mCalc.isHistorical || false,
|
|
680
|
+
dependencies: mCalc.dependencies || [],
|
|
681
|
+
rootDataDependencies: mCalc.rootDataDependencies || [],
|
|
682
|
+
pass: mCalc.pass,
|
|
683
|
+
type: 'meta'
|
|
684
|
+
}
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
successCalcs.push(name);
|
|
688
|
+
} else {
|
|
689
|
+
// Calc ran but returned no data
|
|
690
|
+
successCalcs.push(name);
|
|
376
691
|
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
if (calcClass && typeof calcClass.getSchema === 'function') {
|
|
380
|
-
try { staticSchema = calcClass.getSchema();
|
|
381
|
-
} catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message }); }
|
|
382
|
-
} else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
|
|
383
|
-
if (staticSchema) { schemasToStore.push({ name, category: mCalc.category, schema: staticSchema, metadata: { isHistorical: mCalc.isHistorical || false, dependencies: mCalc.dependencies || [], rootDataDependencies: mCalc.rootDataDependencies || [], pass: mCalc.pass, type: 'meta' } }); }
|
|
384
|
-
success++;
|
|
692
|
+
} catch (e) { logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
|
|
693
|
+
failedCalcs.push({ name, error: e.message });
|
|
385
694
|
}
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
|
|
695
|
+
} // --- End Meta-Calc Loop ---
|
|
696
|
+
|
|
697
|
+
// --- Commit Writes ---
|
|
698
|
+
if (schemasToStore.length > 0) {
|
|
699
|
+
batchStoreSchemas(deps, config, schemasToStore).catch(err => {
|
|
700
|
+
logger.log('WARN', `[SchemaCapture] Non-blocking schema storage failed for ${dStr}`, { errorMessage: err.message });
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`); }
|
|
705
|
+
|
|
389
706
|
for (const collectionName in shardedWrites) {
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
707
|
+
const docs = shardedWrites[collectionName];
|
|
708
|
+
const shardedDocWrites = [];
|
|
709
|
+
for (const docId in docs) {
|
|
710
|
+
const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId);
|
|
711
|
+
const docData = docs[docId];
|
|
712
|
+
docData._completed = true; // Mark as complete
|
|
713
|
+
shardedDocWrites.push({ ref: docRef, data: docData });
|
|
714
|
+
}
|
|
715
|
+
if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`); }
|
|
398
716
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
717
|
+
|
|
718
|
+
// --- Final Verbose Log ---
|
|
719
|
+
const logMetadata = {
|
|
720
|
+
total_expected: calcs.length,
|
|
721
|
+
success_count: successCalcs.length,
|
|
722
|
+
failed_count: failedCalcs.length,
|
|
723
|
+
successful_calcs: successCalcs,
|
|
724
|
+
failed_calcs: failedCalcs
|
|
725
|
+
};
|
|
726
|
+
logger.log( failedCalcs.length === 0 ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}.`,logMetadata );
|
|
403
727
|
}
|
|
404
728
|
|
|
405
|
-
|
|
729
|
+
|
|
730
|
+
module.exports = {
|
|
731
|
+
groupByPass,
|
|
732
|
+
checkRootDataAvailability,
|
|
733
|
+
fetchExistingResults,
|
|
734
|
+
filterCalculations,
|
|
735
|
+
runStandardComputationPass,
|
|
736
|
+
runMetaComputationPass
|
|
737
|
+
};
|