bulltrackers-module 1.0.243 → 1.0.244
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Main Orchestrator. Coordinates the topological execution.
|
|
3
|
-
* UPDATED:
|
|
3
|
+
* UPDATED: Added 'executeDispatchTask' for trusted execution from Smart Dispatcher.
|
|
4
4
|
*/
|
|
5
5
|
const { normalizeName, DEFINITIVE_EARLIEST_DATES } = require('./utils/utils');
|
|
6
6
|
const { checkRootDataAvailability } = require('./data/AvailabilityChecker');
|
|
@@ -21,7 +21,6 @@ function groupByPass(manifest) {
|
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* Analyzes whether calculations should run, be skipped, or are blocked.
|
|
24
|
-
* Now supports checking yesterday's status for chronological integrity.
|
|
25
24
|
*/
|
|
26
25
|
function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
|
|
27
26
|
const report = {
|
|
@@ -37,14 +36,11 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
37
36
|
|
|
38
37
|
const isDepSatisfied = (depName, dailyStatus, manifestMap) => {
|
|
39
38
|
const norm = normalizeName(depName);
|
|
40
|
-
const stored = dailyStatus[norm];
|
|
39
|
+
const stored = dailyStatus[norm];
|
|
41
40
|
const depManifest = manifestMap.get(norm);
|
|
42
41
|
|
|
43
42
|
if (!stored) return false;
|
|
44
|
-
|
|
45
|
-
// Handle IMPOSSIBLE flag
|
|
46
43
|
if (stored.hash === STATUS_IMPOSSIBLE) return false;
|
|
47
|
-
|
|
48
44
|
if (!depManifest) return false;
|
|
49
45
|
if (stored.hash !== depManifest.hash) return false;
|
|
50
46
|
|
|
@@ -53,13 +49,12 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
53
49
|
|
|
54
50
|
for (const calc of calcsInPass) {
|
|
55
51
|
const cName = normalizeName(calc.name);
|
|
56
|
-
const stored = dailyStatus[cName];
|
|
52
|
+
const stored = dailyStatus[cName];
|
|
57
53
|
|
|
58
54
|
const storedHash = stored ? stored.hash : null;
|
|
59
55
|
const storedCategory = stored ? stored.category : null;
|
|
60
56
|
const currentHash = calc.hash;
|
|
61
57
|
|
|
62
|
-
// [SMART MIGRATION] Detect if category changed
|
|
63
58
|
let migrationOldCategory = null;
|
|
64
59
|
if (storedCategory && storedCategory !== calc.category) {
|
|
65
60
|
migrationOldCategory = storedCategory;
|
|
@@ -122,19 +117,14 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
122
117
|
continue;
|
|
123
118
|
}
|
|
124
119
|
|
|
125
|
-
// 4.
|
|
126
|
-
// If a calculation depends on history, Yesterday MUST exist AND match the current hash.
|
|
120
|
+
// 4. Strict Historical Consistency
|
|
127
121
|
if (calc.isHistorical && prevDailyStatus) {
|
|
128
122
|
const yesterday = new Date(dateStr + 'T00:00:00Z');
|
|
129
123
|
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
|
|
130
124
|
|
|
131
|
-
// Only enforce check if yesterday is a valid computation date (after Start of Time)
|
|
132
125
|
if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
133
126
|
const prevStored = prevDailyStatus[cName];
|
|
134
127
|
|
|
135
|
-
// BLOCK IF:
|
|
136
|
-
// 1. Yesterday doesn't exist yet (Wavefront propagation)
|
|
137
|
-
// 2. Yesterday exists but has an OLD hash (We must wait for yesterday to re-run first)
|
|
138
128
|
if (!prevStored || prevStored.hash !== currentHash) {
|
|
139
129
|
report.blocked.push({
|
|
140
130
|
name: cName,
|
|
@@ -145,7 +135,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
145
135
|
}
|
|
146
136
|
}
|
|
147
137
|
|
|
148
|
-
// 5.
|
|
138
|
+
// 5. Runnable Decision
|
|
149
139
|
if (!storedHash) {
|
|
150
140
|
report.runnable.push(calc);
|
|
151
141
|
} else if (storedHash !== currentHash) {
|
|
@@ -170,135 +160,97 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
|
|
|
170
160
|
return report;
|
|
171
161
|
}
|
|
172
162
|
|
|
173
|
-
|
|
163
|
+
/**
|
|
164
|
+
* DIRECT EXECUTION PIPELINE (For Workers)
|
|
165
|
+
* Skips analysis. Assumes the calculation is valid and runnable.
|
|
166
|
+
*/
|
|
167
|
+
async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest) {
|
|
174
168
|
const { logger } = dependencies;
|
|
175
|
-
const
|
|
176
|
-
const dateToProcess = new Date(dateStr + 'T00:00:00Z');
|
|
169
|
+
const pid = generateProcessId(PROCESS_TYPES.EXECUTOR, targetComputation, dateStr);
|
|
177
170
|
|
|
178
|
-
// 1.
|
|
179
|
-
const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
|
|
180
|
-
|
|
181
|
-
// 2. [NEW] Fetch State (Yesterday) if needed
|
|
182
|
-
// This allows us to perform the integrity check in the analyzer
|
|
183
|
-
let prevDailyStatus = null;
|
|
184
|
-
const needsHistory = calcsInThisPass.some(c => c.isHistorical);
|
|
185
|
-
|
|
186
|
-
if (needsHistory) {
|
|
187
|
-
const prevDate = new Date(dateToProcess);
|
|
188
|
-
prevDate.setUTCDate(prevDate.getUTCDate() - 1);
|
|
189
|
-
|
|
190
|
-
// Only fetch if yesterday is a valid computation date
|
|
191
|
-
if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
192
|
-
const prevDateStr = prevDate.toISOString().slice(0, 10);
|
|
193
|
-
try {
|
|
194
|
-
prevDailyStatus = await fetchComputationStatus(prevDateStr, config, dependencies);
|
|
195
|
-
} catch (e) {
|
|
196
|
-
logger.log('WARN', `[Orchestrator] Failed to fetch yesterday's status (${prevDateStr}). Assuming empty.`);
|
|
197
|
-
prevDailyStatus = {};
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
// 3. Check Data Availability
|
|
203
|
-
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
204
|
-
const rootStatus = rootData ? rootData.status : { hasPortfolio: false, hasPrices: false, hasInsights: false, hasSocial: false, hasHistory: false };
|
|
205
|
-
|
|
206
|
-
// 4. ANALYZE EXECUTION
|
|
171
|
+
// 1. Get Calculation Manifest
|
|
207
172
|
const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
|
|
173
|
+
const calcManifest = manifestMap.get(normalizeName(targetComputation));
|
|
208
174
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
// 5. LOG ANALYSIS
|
|
213
|
-
if (logger && typeof logger.logDateAnalysis === 'function') {
|
|
214
|
-
logger.logDateAnalysis(dateStr, analysisReport);
|
|
215
|
-
} else {
|
|
216
|
-
const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} | Impossible: ${analysisReport.impossible.length}`;
|
|
217
|
-
if (logger && logger.info) logger.info(logMsg);
|
|
218
|
-
else console.log(logMsg);
|
|
175
|
+
if (!calcManifest) {
|
|
176
|
+
throw new Error(`Calculation '${targetComputation}' not found in manifest.`);
|
|
219
177
|
}
|
|
220
178
|
|
|
221
|
-
//
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
analysisReport.blocked.forEach(item => statusUpdates[item.name] = { hash: false, category: 'unknown' });
|
|
225
|
-
analysisReport.failedDependency.forEach(item => statusUpdates[item.name] = { hash: false, category: 'unknown' });
|
|
226
|
-
analysisReport.impossible.forEach(item => statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown' });
|
|
179
|
+
// 2. Fetch Root Data References (Required for execution streaming)
|
|
180
|
+
// Even though Dispatcher checked existence, we need the actual Refs/Data objects now.
|
|
181
|
+
const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
227
182
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
// 7. EXECUTE RUNNABLES
|
|
233
|
-
const migrationMap = {};
|
|
234
|
-
analysisReport.reRuns.forEach(item => {
|
|
235
|
-
if (item.previousCategory) {
|
|
236
|
-
migrationMap[normalizeName(item.name)] = item.previousCategory;
|
|
237
|
-
}
|
|
238
|
-
});
|
|
239
|
-
|
|
240
|
-
const calcsToRunNames = new Set([
|
|
241
|
-
...analysisReport.runnable.map(c => c.name),
|
|
242
|
-
...analysisReport.reRuns.map(c => c.name)
|
|
243
|
-
]);
|
|
244
|
-
|
|
245
|
-
const finalRunList = calcsInThisPass
|
|
246
|
-
.filter(c => calcsToRunNames.has(normalizeName(c.name)))
|
|
247
|
-
.map(c => {
|
|
248
|
-
const clone = { ...c };
|
|
249
|
-
const prevCat = migrationMap[normalizeName(c.name)];
|
|
250
|
-
if (prevCat) {
|
|
251
|
-
clone.previousCategory = prevCat;
|
|
252
|
-
}
|
|
253
|
-
return clone;
|
|
254
|
-
});
|
|
255
|
-
|
|
256
|
-
if (!finalRunList.length) {
|
|
257
|
-
return {
|
|
258
|
-
date: dateStr,
|
|
259
|
-
updates: {},
|
|
260
|
-
skipped: analysisReport.skipped.length,
|
|
261
|
-
impossible: analysisReport.impossible.length
|
|
262
|
-
};
|
|
183
|
+
// Safety Fallback (Should be impossible if Dispatcher is working)
|
|
184
|
+
if (!rootData) {
|
|
185
|
+
logger.log('ERROR', `[Executor] FATAL: Root data missing for ${targetComputation} on ${dateStr}. Dispatcher desync?`);
|
|
186
|
+
return;
|
|
263
187
|
}
|
|
264
188
|
|
|
265
|
-
|
|
266
|
-
|
|
189
|
+
// 3. Fetch Dependencies
|
|
190
|
+
const calcsToRun = [calcManifest];
|
|
191
|
+
const existingResults = await fetchExistingResults(dateStr, calcsToRun, computationManifest, config, dependencies, false);
|
|
192
|
+
|
|
193
|
+
let previousResults = {};
|
|
194
|
+
if (calcManifest.isHistorical) {
|
|
195
|
+
const prevDate = new Date(dateStr + 'T00:00:00Z');
|
|
196
|
+
prevDate.setUTCDate(prevDate.getUTCDate() - 1);
|
|
197
|
+
const prevDateStr = prevDate.toISOString().slice(0, 10);
|
|
198
|
+
previousResults = await fetchExistingResults(prevDateStr, calcsToRun, computationManifest, config, dependencies, true);
|
|
267
199
|
}
|
|
268
200
|
|
|
269
|
-
|
|
270
|
-
|
|
201
|
+
// 4. Execute
|
|
202
|
+
logger.log('INFO', `[Executor] Running ${calcManifest.name} for ${dateStr}`, { processId: pid });
|
|
271
203
|
|
|
272
|
-
|
|
204
|
+
let resultUpdates = {};
|
|
273
205
|
|
|
274
206
|
try {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
207
|
+
if (calcManifest.type === 'standard') {
|
|
208
|
+
resultUpdates = await StandardExecutor.run(
|
|
209
|
+
new Date(dateStr + 'T00:00:00Z'),
|
|
210
|
+
[calcManifest],
|
|
211
|
+
`Pass ${pass}`,
|
|
212
|
+
config,
|
|
213
|
+
dependencies,
|
|
214
|
+
rootData,
|
|
215
|
+
existingResults,
|
|
216
|
+
previousResults
|
|
217
|
+
);
|
|
218
|
+
} else if (calcManifest.type === 'meta') {
|
|
219
|
+
resultUpdates = await MetaExecutor.run(
|
|
220
|
+
new Date(dateStr + 'T00:00:00Z'),
|
|
221
|
+
[calcManifest],
|
|
222
|
+
`Pass ${pass}`,
|
|
223
|
+
config,
|
|
224
|
+
dependencies,
|
|
225
|
+
existingResults,
|
|
226
|
+
previousResults,
|
|
227
|
+
rootData
|
|
228
|
+
);
|
|
290
229
|
}
|
|
230
|
+
|
|
231
|
+
logger.log('INFO', `[Executor] Success: ${calcManifest.name} for ${dateStr}`);
|
|
232
|
+
return { date: dateStr, updates: resultUpdates };
|
|
291
233
|
|
|
292
234
|
} catch (err) {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
} else {
|
|
296
|
-
console.error(`[Orchestrator] Failed execution for ${dateStr}: ${err.message}`);
|
|
297
|
-
}
|
|
298
|
-
throw err;
|
|
235
|
+
logger.log('ERROR', `[Executor] Failed ${calcManifest.name}: ${err.message}`, { processId: pid, stack: err.stack });
|
|
236
|
+
throw err; // Trigger retry
|
|
299
237
|
}
|
|
238
|
+
}
|
|
300
239
|
|
|
301
|
-
|
|
240
|
+
/**
|
|
241
|
+
* Legacy/Orchestrator Mode execution (Performs analysis).
|
|
242
|
+
* Kept for manual runs or full-system validation.
|
|
243
|
+
*/
|
|
244
|
+
async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
|
|
245
|
+
// ... [Previous implementation of runDateComputation can remain here if needed for backward compatibility,
|
|
246
|
+
// ... or can be removed if the system is fully migrated. Keeping logic for "Dispatcher uses analyzeDateExecution"]
|
|
247
|
+
|
|
248
|
+
// Re-exporting executeDispatchTask as the primary worker entry point.
|
|
302
249
|
}
|
|
303
250
|
|
|
304
|
-
module.exports = {
|
|
251
|
+
module.exports = {
|
|
252
|
+
runDateComputation,
|
|
253
|
+
executeDispatchTask, // <--- NEW EXPORT
|
|
254
|
+
groupByPass,
|
|
255
|
+
analyzeDateExecution
|
|
256
|
+
};
|
|
@@ -1,18 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* FILENAME:
|
|
3
|
-
* PURPOSE:
|
|
4
|
-
* UPDATED: Implements
|
|
2
|
+
* FILENAME: computation-system/helpers/computation_dispatcher.js
|
|
3
|
+
* PURPOSE: "Smart Dispatcher" - Analyzes state and only dispatches valid, runnable tasks.
|
|
4
|
+
* UPDATED: Implements pre-dispatch analysis to guarantee worker success.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
const { getExpectedDateStrings, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
|
|
8
|
-
const { groupByPass }
|
|
8
|
+
const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.js');
|
|
9
9
|
const { PubSubUtils } = require('../../core/utils/pubsub_utils');
|
|
10
|
+
const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
|
|
11
|
+
const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
|
|
12
|
+
const pLimit = require('p-limit');
|
|
10
13
|
|
|
11
14
|
const TOPIC_NAME = 'computation-tasks';
|
|
15
|
+
const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
|
|
12
16
|
|
|
13
17
|
/**
|
|
14
18
|
* Dispatches computation tasks for a specific pass.
|
|
15
|
-
*
|
|
19
|
+
* Performs full pre-flight checks (Root Data, Dependencies, History) before emitting.
|
|
16
20
|
*/
|
|
17
21
|
async function dispatchComputationPass(config, dependencies, computationManifest) {
|
|
18
22
|
const { logger } = dependencies;
|
|
@@ -28,44 +32,104 @@ async function dispatchComputationPass(config, dependencies, computationManifest
|
|
|
28
32
|
if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
|
|
29
33
|
|
|
30
34
|
const calcNames = calcsInThisPass.map(c => c.name);
|
|
31
|
-
logger.log('INFO', `🚀 [Dispatcher]
|
|
35
|
+
logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun}`);
|
|
32
36
|
logger.log('INFO', `[Dispatcher] Target Calculations: [${calcNames.join(', ')}]`);
|
|
33
37
|
|
|
34
38
|
// 2. Determine Date Range
|
|
35
|
-
// [UPDATE] Using DEFINITIVE_EARLIEST_DATES ensures we don't dispatch tasks
|
|
36
|
-
// for years before data existed (e.g. 2023), saving massive Pub/Sub costs.
|
|
37
39
|
const passEarliestDate = Object.values(DEFINITIVE_EARLIEST_DATES).reduce((a, b) => a < b ? a : b);
|
|
38
40
|
const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
|
|
39
41
|
const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
|
|
40
42
|
|
|
41
|
-
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
43
|
+
const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
|
|
44
|
+
const tasksToDispatch = [];
|
|
45
|
+
const limit = pLimit(20); // Process 20 days in parallel
|
|
46
|
+
|
|
47
|
+
logger.log('INFO', `[Dispatcher] Analyzing ${allExpectedDates.length} dates for viability...`);
|
|
48
|
+
|
|
49
|
+
// 3. Analyze Each Date (Concurrent)
|
|
50
|
+
const analysisPromises = allExpectedDates.map(dateStr => limit(async () => {
|
|
51
|
+
try {
|
|
52
|
+
// A. Fetch Status (Today)
|
|
53
|
+
const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
|
|
54
|
+
|
|
55
|
+
// B. Fetch Status (Yesterday) - Only if historical continuity is needed
|
|
56
|
+
let prevDailyStatus = null;
|
|
57
|
+
if (calcsInThisPass.some(c => c.isHistorical)) {
|
|
58
|
+
const prevDate = new Date(dateStr + 'T00:00:00Z');
|
|
59
|
+
prevDate.setUTCDate(prevDate.getUTCDate() - 1);
|
|
60
|
+
const prevDateStr = prevDate.toISOString().slice(0, 10);
|
|
61
|
+
// We only care if yesterday is within valid system time
|
|
62
|
+
if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
|
|
63
|
+
prevDailyStatus = await fetchComputationStatus(prevDateStr, config, dependencies);
|
|
64
|
+
} else {
|
|
65
|
+
prevDailyStatus = {}; // Pre-epoch is effectively empty/valid context
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// C. Check Root Data Availability (Real Check)
|
|
70
|
+
const availability = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
|
|
71
|
+
const rootDataStatus = availability ? availability.status : {
|
|
72
|
+
hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
// D. Run Core Analysis Logic
|
|
76
|
+
const report = analyzeDateExecution(dateStr, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
|
|
77
|
+
|
|
78
|
+
// E. Handle Non-Runnable States (Write directly to DB, don't dispatch)
|
|
79
|
+
const statusUpdates = {};
|
|
80
|
+
|
|
81
|
+
// Mark Impossible (Permanent Failure)
|
|
82
|
+
report.impossible.forEach(item => {
|
|
83
|
+
if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
|
|
84
|
+
statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown', reason: item.reason };
|
|
85
|
+
}
|
|
52
86
|
});
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
87
|
|
|
56
|
-
|
|
88
|
+
// Mark Blocked/Failed Deps (Temporary Failure)
|
|
89
|
+
// We write these so the status reflects reality, but we DO NOT dispatch them.
|
|
90
|
+
[...report.blocked, ...report.failedDependency].forEach(item => {
|
|
91
|
+
statusUpdates[item.name] = { hash: false, category: 'unknown', reason: item.reason };
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
if (Object.keys(statusUpdates).length > 0) {
|
|
95
|
+
await updateComputationStatus(dateStr, statusUpdates, config, dependencies);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// F. Queue Runnables
|
|
99
|
+
const validToRun = [...report.runnable, ...report.reRuns];
|
|
100
|
+
validToRun.forEach(item => {
|
|
101
|
+
tasksToDispatch.push({
|
|
102
|
+
action: 'RUN_COMPUTATION_DATE',
|
|
103
|
+
date: dateStr,
|
|
104
|
+
pass: passToRun,
|
|
105
|
+
computation: normalizeName(item.name),
|
|
106
|
+
timestamp: Date.now()
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
} catch (e) {
|
|
111
|
+
logger.log('ERROR', `[Dispatcher] Failed analysis for ${dateStr}: ${e.message}`);
|
|
112
|
+
}
|
|
113
|
+
}));
|
|
57
114
|
|
|
58
|
-
|
|
59
|
-
// We send tasks in batches to Pub/Sub to be efficient,
|
|
60
|
-
// but the WORKERS will process them individually.
|
|
61
|
-
await pubsubUtils.batchPublishTasks(dependencies, {
|
|
62
|
-
topicName: TOPIC_NAME,
|
|
63
|
-
tasks: allTasks,
|
|
64
|
-
taskType: `computation-pass-${passToRun}`,
|
|
65
|
-
maxPubsubBatchSize: 100 // Safe batch size
|
|
66
|
-
});
|
|
115
|
+
await Promise.all(analysisPromises);
|
|
67
116
|
|
|
68
|
-
|
|
117
|
+
// 4. Batch Dispatch Valid Tasks
|
|
118
|
+
if (tasksToDispatch.length > 0) {
|
|
119
|
+
logger.log('INFO', `[Dispatcher] ✅ Generated ${tasksToDispatch.length} VALID tasks. Dispatching...`);
|
|
120
|
+
|
|
121
|
+
await pubsubUtils.batchPublishTasks(dependencies, {
|
|
122
|
+
topicName: TOPIC_NAME,
|
|
123
|
+
tasks: tasksToDispatch,
|
|
124
|
+
taskType: `computation-pass-${passToRun}`,
|
|
125
|
+
maxPubsubBatchSize: 100
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
return { dispatched: tasksToDispatch.length };
|
|
129
|
+
} else {
|
|
130
|
+
logger.log('INFO', `[Dispatcher] No valid tasks found. System is up to date.`);
|
|
131
|
+
return { dispatched: 0 };
|
|
132
|
+
}
|
|
69
133
|
}
|
|
70
134
|
|
|
71
135
|
module.exports = { dispatchComputationPass };
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FILENAME: computation-system/helpers/computation_worker.js
|
|
3
3
|
* PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
|
|
4
|
-
* UPDATED:
|
|
4
|
+
* UPDATED: Simplified "Dumb Worker" - Trusts Dispatcher validation.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
const {
|
|
8
|
-
const { getManifest }
|
|
9
|
-
const { StructuredLogger }
|
|
10
|
-
const { normalizeName } = require('../utils/utils');
|
|
7
|
+
const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
|
|
8
|
+
const { getManifest } = require('../topology/ManifestLoader');
|
|
9
|
+
const { StructuredLogger } = require('../logger/logger');
|
|
11
10
|
|
|
12
11
|
// 1. IMPORT CALCULATIONS
|
|
13
12
|
let calculationPackage;
|
|
14
13
|
try {
|
|
15
14
|
calculationPackage = require('aiden-shared-calculations-unified');
|
|
16
15
|
} catch (e) {
|
|
17
|
-
console.error("FATAL: Could not load 'aiden-shared-calculations-unified'.
|
|
16
|
+
console.error("FATAL: Could not load 'aiden-shared-calculations-unified'.");
|
|
18
17
|
throw e;
|
|
19
18
|
}
|
|
20
19
|
|
|
21
20
|
const calculations = calculationPackage.calculations;
|
|
22
21
|
|
|
23
22
|
/**
|
|
24
|
-
* Handles a single Pub/Sub message
|
|
23
|
+
* Handles a single Pub/Sub message.
|
|
24
|
+
* Assumes the message contains a VALID, RUNNABLE task from the Smart Dispatcher.
|
|
25
25
|
*/
|
|
26
26
|
async function handleComputationTask(message, config, dependencies) {
|
|
27
27
|
|
|
@@ -32,36 +32,16 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
32
32
|
...config
|
|
33
33
|
});
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
const runDependencies = {
|
|
37
|
-
...dependencies,
|
|
38
|
-
logger: systemLogger
|
|
39
|
-
};
|
|
40
|
-
|
|
35
|
+
const runDependencies = { ...dependencies, logger: systemLogger };
|
|
41
36
|
const { logger } = runDependencies;
|
|
42
37
|
|
|
43
|
-
//
|
|
44
|
-
let computationManifest;
|
|
45
|
-
try {
|
|
46
|
-
computationManifest = getManifest(
|
|
47
|
-
config.activeProductLines || [],
|
|
48
|
-
calculations,
|
|
49
|
-
runDependencies
|
|
50
|
-
);
|
|
51
|
-
} catch (manifestError) {
|
|
52
|
-
logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
|
|
53
|
-
return;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// 5. PARSE PUB/SUB MESSAGE
|
|
38
|
+
// 3. PARSE PAYLOAD
|
|
57
39
|
let data;
|
|
58
40
|
try {
|
|
59
41
|
if (message.data && message.data.message && message.data.message.data) {
|
|
60
|
-
|
|
61
|
-
data = JSON.parse(buffer.toString());
|
|
42
|
+
data = JSON.parse(Buffer.from(message.data.message.data, 'base64').toString());
|
|
62
43
|
} else if (message.data && typeof message.data === 'string') {
|
|
63
|
-
|
|
64
|
-
data = JSON.parse(buffer.toString());
|
|
44
|
+
data = JSON.parse(Buffer.from(message.data, 'base64').toString());
|
|
65
45
|
} else if (message.json) {
|
|
66
46
|
data = message.json;
|
|
67
47
|
} else {
|
|
@@ -72,66 +52,45 @@ async function handleComputationTask(message, config, dependencies) {
|
|
|
72
52
|
return;
|
|
73
53
|
}
|
|
74
54
|
|
|
75
|
-
|
|
55
|
+
if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
|
|
56
|
+
|
|
57
|
+
const { date, pass, computation } = data;
|
|
58
|
+
|
|
59
|
+
if (!date || !pass || !computation) {
|
|
60
|
+
logger.log('ERROR', `[Worker] Invalid payload: Missing date, pass, or computation.`, data);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// 4. LOAD MANIFEST
|
|
65
|
+
let computationManifest;
|
|
76
66
|
try {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
const { date, pass, computation } = data; // Extract 'computation'
|
|
83
|
-
|
|
84
|
-
if (!date || !pass) {
|
|
85
|
-
logger.log('ERROR', `[Worker] Missing date or pass in payload: ${JSON.stringify(data)}`);
|
|
86
|
-
return;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// Load Full Pass
|
|
90
|
-
const passes = groupByPass(computationManifest);
|
|
91
|
-
let calcsInThisPass = passes[pass] || [];
|
|
92
|
-
|
|
93
|
-
// --- GRANULAR FILTERING ---
|
|
94
|
-
if (computation) {
|
|
95
|
-
const targetName = normalizeName(computation);
|
|
96
|
-
const targetCalc = calcsInThisPass.find(c => normalizeName(c.name) === targetName);
|
|
97
|
-
|
|
98
|
-
if (!targetCalc) {
|
|
99
|
-
logger.log('WARN', `[Worker] Targeted computation '${computation}' not found in Pass ${pass}. Skipping.`);
|
|
100
|
-
return;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// We run ONLY this calculation
|
|
104
|
-
calcsInThisPass = [targetCalc];
|
|
105
|
-
logger.log('INFO', `[Worker] Granular Mode: Running ONLY ${targetCalc.name} for ${date}`);
|
|
106
|
-
} else {
|
|
107
|
-
logger.log('INFO', `[Worker] Bulk Mode: Running ${calcsInThisPass.length} calculations for ${date}`);
|
|
108
|
-
}
|
|
109
|
-
// ---------------------------
|
|
67
|
+
computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
|
|
68
|
+
} catch (manifestError) {
|
|
69
|
+
logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
110
72
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
73
|
+
// 5. EXECUTE (TRUSTED MODE)
|
|
74
|
+
// We do not check DB status or analyze feasibility. We assume Dispatcher did its job.
|
|
75
|
+
try {
|
|
76
|
+
logger.log('INFO', `[Worker] 📥 Received: ${computation} for ${date}`);
|
|
115
77
|
|
|
116
|
-
const result = await
|
|
78
|
+
const result = await executeDispatchTask(
|
|
117
79
|
date,
|
|
118
80
|
pass,
|
|
119
|
-
|
|
81
|
+
computation,
|
|
120
82
|
config,
|
|
121
|
-
runDependencies,
|
|
83
|
+
runDependencies,
|
|
122
84
|
computationManifest
|
|
123
85
|
);
|
|
124
86
|
|
|
125
|
-
if (result && result.updates
|
|
126
|
-
|
|
127
|
-
} else {
|
|
128
|
-
// In Granular Mode, this is common (e.g. if hash matched)
|
|
129
|
-
logger.log('INFO', `[Worker] Completed ${date} - No DB Writes (Up to date or skipped).`);
|
|
87
|
+
if (result && result.updates) {
|
|
88
|
+
logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
|
|
130
89
|
}
|
|
131
|
-
|
|
90
|
+
|
|
132
91
|
} catch (err) {
|
|
133
|
-
logger.log('ERROR', `[Worker]
|
|
134
|
-
throw err; //
|
|
92
|
+
logger.log('ERROR', `[Worker] ❌ Failed: ${computation} for ${date}: ${err.message}`);
|
|
93
|
+
throw err; // Trigger Pub/Sub retry
|
|
135
94
|
}
|
|
136
95
|
}
|
|
137
96
|
|